Commit c00e620f authored by Jyotish P's avatar Jyotish P
Browse files

Add setup and submit utility scripts

parent 6dc096e3
#!/bin/bash
log_info() {
echo -e "\033[0;36m$@\033[0m"
}
log_success() {
echo -e "\033[0;32m$@\033[0m"
}
log_normal() {
echo -e "$@"
}
log_error() {
echo -e "\033[0;31m$@\033[0m"
}
#!/usr/bin/env python3
print("Evaluator script to test predictions locally to be added here.")
#!/bin/bash
set -e
MINICONDA_DOWNLOAD_PATH="/tmp/miniconda.sh"
REPO_ROOT_DIR=$(git rev-parse --show-toplevel)
SCRIPTS_DIR="${REPO_ROOT_DIR}/utility"
NETHACK_PYPI_PACKAGE_NAME="git+https://github.com/facebookresearch/nle.git"
source "${SCRIPTS_DIR}/logging.sh"
_download_miniconda() {
log_info "Downloading Miniconda..."
miniconda_base_url="https://repo.anaconda.com/miniconda/Miniconda3-latest-"
case `uname` in
"Linux")
miniconda_file="Linux-x86_64.sh"
;;
"Darwin")
miniconda_file="MacOSX-x86_64.sh"
;;
*)
log_error "Sorry, we don't support this platform :("
;;
esac
wget -O "${MINICONDA_DOWNLOAD_PATH}" "${miniconda_base_url}${miniconda_file}"
log_success "Downloaded Miniconda!"
}
download_miniconda() {
if [ -f "${MINICONDA_DOWNLOAD_PATH}" ]; then
return
fi
_download_miniconda
}
_install_miniconda() {
log_info "Installing Miniconda..."
bash ${MINICONDA_DOWNLOAD_PATH} -b -p ${HOME}/miniconda3
. ${HOME}/miniconda3/etc/profile.d/conda.sh
conda init
}
install_miniconda() {
if which conda 2> /dev/null 1>&2; then
log_info "Found an existing conda installation!"
return
fi
download_miniconda
_install_miniconda
}
_setup_nethack_conda_env() {
log_info "Creating nethack conda environment..."
source "$(conda info --base)/etc/profile.d/conda.sh"
conda create -n nethack -y
conda activate nethack
conda install python cmake -y
log_success "Created nethack conda environment!"
}
_install_nethack() {
log_info "Installing NetHack gym environment..."
source "$(conda info --base)/etc/profile.d/conda.sh"
conda activate nethack
conda install -y numpy scikit-learn
pip install -U "${NETHACK_PYPI_PACKAGE_NAME}" aicrowd-gym
pip install -r "${REPO_ROOT_DIR}/requirements.txt"
log_success "Installed NetHack gym environment!"
}
setup_nethack_conda_env() {
if conda activate nethack 2> /dev/null 1>&2; then
log_info "Re-using existing nethack conda environment..."
else
_setup_nethack_conda_env
fi
_install_nethack
}
install_miniconda
setup_nethack_conda_env
log_info "You can start using NetHack conda environment by running"
log_normal "\`conda activate nethack\`"
log_info "**Note:** Please restart your terminal if you did not have conda previously installed."
#!/bin/bash
set -e
REPO_ROOT_DIR=$(git rev-parse --show-toplevel)
SCRIPTS_DIR="${REPO_ROOT_DIR}/utility"
source "${SCRIPTS_DIR}/logging.sh"
print_usage() {
cat << USAGE
Usage: ./utility/submit.sh "impala-ppo-v0.1"
USAGE
}
bad_remote_message() {
log_normal "AIcrowd remote not found"
log_error "Please run \`git remote add aicrowd git@gitlab.aicrowd.com:<username>/<repo>.git\` and rerun this command"
exit 1
}
check_remote() {
log_info Checking git remote settings...
bad_remotes=(
git@gitlab.aicrowd.com:nethack/neurips-2021-the-nethack-challenge.git
http://gitlab.aicrowd.com/nethack/neurips-2021-the-nethack-challenge.git
)
for bad_remote in $bad_remotes; do
if git remote -v | grep "$bad_remote" > /dev/null; then
bad_remote_message
fi
done
if ! git remote -v | grep "gitlab.aicrowd.com"; then
bad_remote_message
fi
}
setup_lfs() {
git lfs install
HTTPS_REMOTE=$(git remote -v | grep gitlab.aicrowd.com | head -1 | awk '{print $2}' | sed 's|git@gitlab.aicrowd.com:|https://gitlab.aicrowd.com|g')
git config lfs.$HTTPS_REMOTE/info/lfs.locksverify false
find . -type f -size +5M -exec git lfs track {} &> /dev/null \;
git add .gitattributes
}
setup_commits() {
REMOTE=$(git remote -v | grep gitlab.aicrowd.com | head -1 | awk '{print $1}')
TAG=$(echo "$@" | sed 's/ /-/g')
git add --all
git commit -m "Changes for submission-$TAG"
git tag -am "submission-$TAG" "submission-$TAG" || (log_error "There is another submission with the same description. Please give a different description." && exit 1)
git push -f $REMOTE master
git push -f $REMOTE "submission-$TAG"
}
submit() {
check_remote
setup_lfs
setup_commits
}
if [[ $# -lt 1 ]]; then
print_usage
exit 1
fi
submit "$@"
#!/bin/bash
set -e
REPO_ROOT_DIR=$(git rev-parse --show-toplevel)
SCRIPTS_DIR="${REPO_ROOT_DIR}/utility"
source "${SCRIPTS_DIR}/logging.sh"
teardown_nethack_conda_env() {
log_info "Removing nethack conda environment..."
source "$(conda info --base)/etc/profile.d/conda.sh"
conda env remove -n nethack
log_success "Removed nethack conda environment!"
}
teardown_procgen_env
import os, sys
import requests
import zipfile
sys.path.append(os.path.dirname(os.path.realpath(os.getcwd())))
sys.path.append(os.path.realpath(os.getcwd()))
DATASET_FILE_NAME = 'download.zip'
DATASET_DOWNLOAD_URL = 'https://zenodo.org/record/3270814/files/MUSDB18-7-WAV.zip?download=1'
DATASET_FULL_DOWNLOAD_URL = 'https://zenodo.org/record/3338373/files/musdb18hq.zip?download=1'
def download_dataset(full=False):
dn_url = DATASET_DOWNLOAD_URL
if full:
dn_url = DATASET_FULL_DOWNLOAD_URL
r = requests.get(dn_url, stream=True)
with open(DATASET_FILE_NAME, 'wb') as fd:
for chunk in r.iter_content(chunk_size=256):
fd.write(chunk)
def unzip_dataset():
with zipfile.ZipFile(DATASET_FILE_NAME, 'r') as zip_ref:
zip_ref.extractall('data/')
def cleanup():
os.remove(DATASET_FILE_NAME)
def verify_dataset():
assert os.path.isdir("data/train") and os.path.isdir("data/test"), "Dataset folder not found"
assert os.path.isdir("data/train/Hollow Ground - Left Blind"), "Random song check in training folder failed"
assert os.path.isdir("data/test/Louis Cressy Band - Good Time"), "Random song check in testing folder failed"
def move_to_git_root():
if not os.path.exists(os.path.join(os.getcwd(), ".git")):
os.chdir("..")
assert os.path.exists(os.path.join(os.getcwd(), ".git")), "Unable to reach to repository root"
if __name__ == "__main__":
move_to_git_root()
try:
verify_dataset()
except AssertionError:
print("Dataset not found...")
option = input("Download full dataset (y/Y) or 7s dataset (n/N)? ")
if option.lower() == 'y':
print("Downloading full dataset...")
download_dataset(full=True)
else:
print("Downloading 7s dataset...")
download_dataset()
print("Unzipping the dataset...")
unzip_dataset()
print("Cleaning up...")
#cleanup()
print("Verifying the dataset...")
verify_dataset()
print("Done.")
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment