submission_config.py

+from agents.custom_agent import CustomAgent
+from agents.torchbeast_agent import TorchBeastAgent
+
+from envs.wrappers import addtimelimitwrapper_fn
+
+################################################
+#         Import your own agent code           #
+#      Set Submision_Agent to your agent       #
+#    Set NUM_PARALLEL_ENVIRONMENTS as needed   #
+#  Set submission_env_make_fn to your wrappers #
+#        Test with local_evaluation.py         #
+################################################
+
+
+class SubmissionConfig:
+    ## Add your own agent class
+    # AGENT = CustomAgent
+    AGENT = TorchBeastAgent
+
+
+    ## Change the NUM_ENVIRONMENTS as you need
+    ## for example reduce it if your GPU doesn't fit
+    ## Increasing above 32 is not advisable for the Nethack Challenge 2021
+    NUM_ENVIRONMENTS = 32
+
+
+    ## Add a function that creates your nethack env
+    ## Mainly this is to add wrappers
+    ## Add your wrappers to envs/wrappers.py and change the name here
+    ## IMPORTANT: Don't "call" the function, only provide the name
+    MAKE_ENV_FN = addtimelimitwrapper_fn
+
+
+class TestEvaluationConfig:
+    # Change this to locally check a different number of rollouts
+    # The AIcrowd submission evaluator will not use this
+    # It is only for your local evaluation
+    NUM_EPISODES = 512

test_submission.py

+## This file is intended to emulate the evaluation on AIcrowd
+
+# IMPORTANT - Differences to expect
+# * All the environment's functions are not available
+# * The run might be slower than your local run
+# * Resources might vary from your local machine
+
+import numpy as np
+
+from submission_config import SubmissionConfig, TestEvaluationConfig
+
+from rollout import run_batched_rollout
+from envs.batched_env import BatchedEnv
+
+
+def evaluate():
+    env_make_fn = SubmissionConfig.MAKE_ENV_FN
+    num_envs = SubmissionConfig.NUM_ENVIRONMENTS
+    Agent = SubmissionConfig.AGENT
+
+    num_episodes = TestEvaluationConfig.NUM_EPISODES
+
+    batched_env = BatchedEnv(env_make_fn=env_make_fn, num_envs=num_envs)
+
+    agent = Agent(num_envs, batched_env.num_actions)
+
+    ascensions, scores = run_batched_rollout(num_episodes, batched_env, agent)
+    print(
+        f"Ascensions: {ascensions} "
+        f"Median Score: {np.median(scores)}, "
+        f"Mean Score: {np.mean(scores)}"
+    )
+
+
+if __name__ == "__main__":
+    evaluate()

utility/docker_build.sh

-#!/bin/bash
-
-if [ -e environ_secret.sh ]
-then
-    source utility/environ_secret.sh
-else
-    source utility/environ.sh
-fi
-
-if ! [ -x "$(command -v aicrowd-repo2docker)" ]; then
-  echo 'Error: aicrowd-repo2docker is not installed.' >&2
-  echo 'Please install it using requirements.txt or pip install -U aicrowd-repo2docker' >&2
-  exit 1
-fi
-
-# Expected Env variables : in environ.sh
-
-REPO2DOCKER="$(which aicrowd-repo2docker)"
-
-sudo ${REPO2DOCKER} --no-run \
-  --user-id 1001 \
-  --user-name aicrowd \
-  --image-name ${IMAGE_NAME}:${IMAGE_TAG} \
-  --debug .

utility/docker_run.sh

-#!/bin/bash
-# This script run your submission inside a docker image, this is identical in termrs of 
-# how your code will be executed on AIcrowd platform
-
-if [ -e environ_secret.sh ]
-then
-    echo "Note: Gathering environment variables from environ_secret.sh"
-    source utility/environ_secret.sh
-else
-    echo "Note: Gathering environment variables from environ.sh"
-    source utility/environ.sh
-fi
-
-# Skip building docker image on run, by default each run means new docker image build
-if [[ " $@ " =~ " --no-build " ]]; then
-    echo "Skipping docker image build"
-else
-    echo "Building docker image, for skipping docker image build use \"--no-build\""
-    ./utility/docker_build.sh
-fi
-
-# Expected Env variables : in environ.sh
-sudo docker run \
---net=host \
---user 0 \
--e AICROWD_IS_GRADING=True \
--e AICROWD_DEBUG_MODE=True \
--it ${IMAGE_NAME}:${IMAGE_TAG} \
-/bin/bash
\ No newline at end of file

utility/environ.sh

-#!/bin/bash
-
-export IMAGE_NAME="aicrowd/music-demixing-challenge"
-export IMAGE_TAG="local"

utility/parser.py

-#!/usr/bin/env python3
-print("Evaluator script to test predictions locally to be added here.")

utility/submit.sh

+#!/bin/bash
+
+set -e
+
+REPO_ROOT_DIR=$(git rev-parse --show-toplevel)
+SCRIPTS_DIR="${REPO_ROOT_DIR}/utility"
+
+source "${SCRIPTS_DIR}/logging.sh"
+
+
+print_usage() {
+cat << USAGE
+Usage: ./utility/submit.sh "impala-ppo-v0.1"
+USAGE
+}
+
+
+bad_remote_message() {
+  log_normal "AIcrowd remote not found"
+  log_error "Please run \`git remote add aicrowd git@gitlab.aicrowd.com:<username>/<repo>.git\` and rerun this command"
+  exit 1
+}
+
+check_remote() {
+  log_info Checking git remote settings...
+
+  bad_remotes=(
+    git@gitlab.aicrowd.com:nethack/neurips-2021-the-nethack-challenge.git
+    http://gitlab.aicrowd.com/nethack/neurips-2021-the-nethack-challenge.git
+  )
+  for bad_remote in $bad_remotes; do
+    if git remote -v | grep "$bad_remote" > /dev/null; then
+      bad_remote_message
+    fi
+  done
+
+  if ! git remote -v | grep "gitlab.aicrowd.com"; then
+    bad_remote_message
+  fi
+}
+
+
+setup_lfs() {
+  git lfs install
+  HTTPS_REMOTE=$(git remote -v | grep gitlab.aicrowd.com | head -1 | awk '{print $2}' | sed 's|git@gitlab.aicrowd.com:|https://gitlab.aicrowd.com|g')
+  git config lfs.$HTTPS_REMOTE/info/lfs.locksverify false
+  find . -type f -size +5M -exec git lfs track {} &> /dev/null \;
+  git add .gitattributes
+}
+
+
+setup_commits() {
+  REMOTE=$(git remote -v | grep gitlab.aicrowd.com | head -1 | awk '{print $1}')
+  TAG=$(echo "$@" | sed 's/ /-/g')
+  git add --all
+  git commit -m "Changes for submission-$TAG"
+  git tag -am "submission-$TAG" "submission-$TAG" || (log_error "There is another submission with the same description. Please give a different description." && exit 1)
+  git push -f $REMOTE master
+  git push -f $REMOTE "submission-$TAG"
+}
+
+
+submit() {
+  check_remote
+  setup_lfs
+  setup_commits
+}
+  
+
+
+if [[ $# -lt 1 ]]; then
+  print_usage
+  exit 1
+fi
+
+submit "$@"

utility/verify_or_download_data.py

-import os, sys
-import requests
-import zipfile
-
-sys.path.append(os.path.dirname(os.path.realpath(os.getcwd())))
-sys.path.append(os.path.realpath(os.getcwd()))
-
-DATASET_FILE_NAME = 'download.zip'
-DATASET_DOWNLOAD_URL = 'https://zenodo.org/record/3270814/files/MUSDB18-7-WAV.zip?download=1'
-DATASET_FULL_DOWNLOAD_URL = 'https://zenodo.org/record/3338373/files/musdb18hq.zip?download=1'
-
-def download_dataset(full=False):
-    dn_url = DATASET_DOWNLOAD_URL
-    if full:
-        dn_url = DATASET_FULL_DOWNLOAD_URL
-    r = requests.get(dn_url, stream=True)
-    with open(DATASET_FILE_NAME, 'wb') as fd:
-        for chunk in r.iter_content(chunk_size=256):
-            fd.write(chunk)
-
-def unzip_dataset():
-    with zipfile.ZipFile(DATASET_FILE_NAME, 'r') as zip_ref:
-        zip_ref.extractall('data/')
-
-def cleanup():
-    os.remove(DATASET_FILE_NAME)
-
-def verify_dataset():
-    assert os.path.isdir("data/train") and os.path.isdir("data/test"), "Dataset folder not found"
-    assert os.path.isdir("data/train/Hollow Ground - Left Blind"), "Random song check in training folder failed"
-    assert os.path.isdir("data/test/Louis Cressy Band - Good Time"), "Random song check in testing folder failed"
-
-def move_to_git_root():
-    if not os.path.exists(os.path.join(os.getcwd(), ".git")):
-        os.chdir("..")
-    assert os.path.exists(os.path.join(os.getcwd(), ".git")), "Unable to reach to repository root"
-
-if __name__ == "__main__":
-    move_to_git_root()
-    try:
-        verify_dataset()
-    except AssertionError:
-        print("Dataset not found...")
-        option = input("Download full dataset (y/Y) or 7s dataset (n/N)? ")
-        if option.lower() == 'y':
-            print("Downloading full dataset...")
-            download_dataset(full=True)
-        else:
-            print("Downloading 7s dataset...")
-            download_dataset()
-        print("Unzipping the dataset...")
-        unzip_dataset()
-        print("Cleaning up...")
-        #cleanup()
-        print("Verifying the dataset...")
-        verify_dataset()
-        print("Done.")