From 6fd44048f4037dab9daaa5a03589512748cf2eba Mon Sep 17 00:00:00 2001
From: Dipam Chakraborty <dipam@aicrowd.com>
Date: Wed, 9 Jun 2021 10:49:13 +0000
Subject: [PATCH] change utility folder
---
utility/docker_build.sh | 24 ----------
utility/docker_run.sh | 29 ------------
utility/environ.sh | 4 --
utility/parser.py | 2 -
utility/submit.sh | 76 ++++++++++++++++++++++++++++++
utility/verify_or_download_data.py | 57 ----------------------
6 files changed, 76 insertions(+), 116 deletions(-)
delete mode 100755 utility/docker_build.sh
delete mode 100755 utility/docker_run.sh
delete mode 100644 utility/environ.sh
delete mode 100644 utility/parser.py
create mode 100644 utility/submit.sh
delete mode 100644 utility/verify_or_download_data.py
diff --git a/utility/docker_build.sh b/utility/docker_build.sh
deleted file mode 100755
index c578aec..0000000
--- a/utility/docker_build.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-if [ -e environ_secret.sh ]
-then
- source utility/environ_secret.sh
-else
- source utility/environ.sh
-fi
-
-if ! [ -x "$(command -v aicrowd-repo2docker)" ]; then
- echo 'Error: aicrowd-repo2docker is not installed.' >&2
- echo 'Please install it using requirements.txt or pip install -U aicrowd-repo2docker' >&2
- exit 1
-fi
-
-# Expected Env variables : in environ.sh
-
-REPO2DOCKER="$(which aicrowd-repo2docker)"
-
-sudo ${REPO2DOCKER} --no-run \
- --user-id 1001 \
- --user-name aicrowd \
- --image-name ${IMAGE_NAME}:${IMAGE_TAG} \
- --debug .
diff --git a/utility/docker_run.sh b/utility/docker_run.sh
deleted file mode 100755
index ee426d7..0000000
--- a/utility/docker_run.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-# This script run your submission inside a docker image, this is identical in termrs of
-# how your code will be executed on AIcrowd platform
-
-if [ -e environ_secret.sh ]
-then
- echo "Note: Gathering environment variables from environ_secret.sh"
- source utility/environ_secret.sh
-else
- echo "Note: Gathering environment variables from environ.sh"
- source utility/environ.sh
-fi
-
-# Skip building docker image on run, by default each run means new docker image build
-if [[ " $@ " =~ " --no-build " ]]; then
- echo "Skipping docker image build"
-else
- echo "Building docker image, for skipping docker image build use \"--no-build\""
- ./utility/docker_build.sh
-fi
-
-# Expected Env variables : in environ.sh
-sudo docker run \
---net=host \
---user 0 \
--e AICROWD_IS_GRADING=True \
--e AICROWD_DEBUG_MODE=True \
--it ${IMAGE_NAME}:${IMAGE_TAG} \
-/bin/bash
\ No newline at end of file
diff --git a/utility/environ.sh b/utility/environ.sh
deleted file mode 100644
index 4b28fca..0000000
--- a/utility/environ.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-export IMAGE_NAME="aicrowd/music-demixing-challenge"
-export IMAGE_TAG="local"
diff --git a/utility/parser.py b/utility/parser.py
deleted file mode 100644
index 34debfd..0000000
--- a/utility/parser.py
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/usr/bin/env python3
-print("Evaluator script to test predictions locally to be added here.")
diff --git a/utility/submit.sh b/utility/submit.sh
new file mode 100644
index 0000000..1231cff
--- /dev/null
+++ b/utility/submit.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+
+set -e
+
+REPO_ROOT_DIR=$(git rev-parse --show-toplevel)
+SCRIPTS_DIR="${REPO_ROOT_DIR}/utility"
+
+source "${SCRIPTS_DIR}/logging.sh"
+
+
+print_usage() {
+cat << USAGE
+Usage: ./utility/submit.sh "impala-ppo-v0.1"
+USAGE
+}
+
+
+bad_remote_message() {
+ log_normal "AIcrowd remote not found"
+ log_error "Please run \`git remote add aicrowd git@gitlab.aicrowd.com:<username>/<repo>.git\` and rerun this command"
+ exit 1
+}
+
+check_remote() {
+ log_info Checking git remote settings...
+
+ bad_remotes=(
+ git@gitlab.aicrowd.com:nethack/neurips-2021-the-nethack-challenge.git
+ http://gitlab.aicrowd.com/nethack/neurips-2021-the-nethack-challenge.git
+ )
+ for bad_remote in $bad_remotes; do
+ if git remote -v | grep "$bad_remote" > /dev/null; then
+ bad_remote_message
+ fi
+ done
+
+ if ! git remote -v | grep "gitlab.aicrowd.com"; then
+ bad_remote_message
+ fi
+}
+
+
+setup_lfs() {
+ git lfs install
+ HTTPS_REMOTE=$(git remote -v | grep gitlab.aicrowd.com | head -1 | awk '{print $2}' | sed 's|git@gitlab.aicrowd.com:|https://gitlab.aicrowd.com|g')
+ git config lfs.$HTTPS_REMOTE/info/lfs.locksverify false
+ find . -type f -size +5M -exec git lfs track {} &> /dev/null \;
+ git add .gitattributes
+}
+
+
+setup_commits() {
+ REMOTE=$(git remote -v | grep gitlab.aicrowd.com | head -1 | awk '{print $1}')
+ TAG=$(echo "$@" | sed 's/ /-/g')
+ git add --all
+ git commit -m "Changes for submission-$TAG"
+ git tag -am "submission-$TAG" "submission-$TAG" || (log_error "There is another submission with the same description. Please give a different description." && exit 1)
+ git push -f $REMOTE master
+ git push -f $REMOTE "submission-$TAG"
+}
+
+
+submit() {
+ check_remote
+ setup_lfs
+ setup_commits
+}
+
+
+
+if [[ $# -lt 1 ]]; then
+ print_usage
+ exit 1
+fi
+
+submit "$@"
diff --git a/utility/verify_or_download_data.py b/utility/verify_or_download_data.py
deleted file mode 100644
index c72395d..0000000
--- a/utility/verify_or_download_data.py
+++ /dev/null
@@ -1,57 +0,0 @@
-import os, sys
-import requests
-import zipfile
-
-sys.path.append(os.path.dirname(os.path.realpath(os.getcwd())))
-sys.path.append(os.path.realpath(os.getcwd()))
-
-DATASET_FILE_NAME = 'download.zip'
-DATASET_DOWNLOAD_URL = 'https://zenodo.org/record/3270814/files/MUSDB18-7-WAV.zip?download=1'
-DATASET_FULL_DOWNLOAD_URL = 'https://zenodo.org/record/3338373/files/musdb18hq.zip?download=1'
-
-def download_dataset(full=False):
- dn_url = DATASET_DOWNLOAD_URL
- if full:
- dn_url = DATASET_FULL_DOWNLOAD_URL
- r = requests.get(dn_url, stream=True)
- with open(DATASET_FILE_NAME, 'wb') as fd:
- for chunk in r.iter_content(chunk_size=256):
- fd.write(chunk)
-
-def unzip_dataset():
- with zipfile.ZipFile(DATASET_FILE_NAME, 'r') as zip_ref:
- zip_ref.extractall('data/')
-
-def cleanup():
- os.remove(DATASET_FILE_NAME)
-
-def verify_dataset():
- assert os.path.isdir("data/train") and os.path.isdir("data/test"), "Dataset folder not found"
- assert os.path.isdir("data/train/Hollow Ground - Left Blind"), "Random song check in training folder failed"
- assert os.path.isdir("data/test/Louis Cressy Band - Good Time"), "Random song check in testing folder failed"
-
-def move_to_git_root():
- if not os.path.exists(os.path.join(os.getcwd(), ".git")):
- os.chdir("..")
- assert os.path.exists(os.path.join(os.getcwd(), ".git")), "Unable to reach to repository root"
-
-if __name__ == "__main__":
- move_to_git_root()
- try:
- verify_dataset()
- except AssertionError:
- print("Dataset not found...")
- option = input("Download full dataset (y/Y) or 7s dataset (n/N)? ")
- if option.lower() == 'y':
- print("Downloading full dataset...")
- download_dataset(full=True)
- else:
- print("Downloading 7s dataset...")
- download_dataset()
- print("Unzipping the dataset...")
- unzip_dataset()
- print("Cleaning up...")
- #cleanup()
- print("Verifying the dataset...")
- verify_dataset()
- print("Done.")
--
GitLab