Adding nomic sleep time
Browse files- src/build_nomic.py +8 -4
src/build_nomic.py
CHANGED
|
@@ -15,6 +15,7 @@ from src.my_logger import setup_logger
|
|
| 15 |
|
| 16 |
NOMIC_KEY = os.getenv('NOMIC_KEY')
|
| 17 |
nomic.login(NOMIC_KEY)
|
|
|
|
| 18 |
logger = setup_logger(__name__)
|
| 19 |
|
| 20 |
# Regex to extract subreddit
|
|
@@ -60,10 +61,13 @@ def delete_old_nomic():
|
|
| 60 |
atlas_id = ac._get_dataset_by_slug_identifier("derek2/boru-subreddit-neural-search")['id']
|
| 61 |
ac._delete_project_by_id(atlas_id)
|
| 62 |
logger.info(f"Succeeded in deleting old version of nomic Atlas.")
|
| 63 |
-
|
| 64 |
-
time
|
| 65 |
-
|
| 66 |
-
|
|
|
|
|
|
|
|
|
|
| 67 |
|
| 68 |
|
| 69 |
def preprocess_markdown(text):
|
|
|
|
| 15 |
|
| 16 |
NOMIC_KEY = os.getenv('NOMIC_KEY')
|
| 17 |
nomic.login(NOMIC_KEY)
|
| 18 |
+
sleep_time = int(os.getenv('NOMIC_SLEEP_TIME', 60))
|
| 19 |
logger = setup_logger(__name__)
|
| 20 |
|
| 21 |
# Regex to extract subreddit
|
|
|
|
| 61 |
atlas_id = ac._get_dataset_by_slug_identifier("derek2/boru-subreddit-neural-search")['id']
|
| 62 |
ac._delete_project_by_id(atlas_id)
|
| 63 |
logger.info(f"Succeeded in deleting old version of nomic Atlas.")
|
| 64 |
+
|
| 65 |
+
# Get sleep time from environment variable
|
| 66 |
+
logger.info(f"Sleeping for {sleep_time}s to wait for old version deletion on the server-side")
|
| 67 |
+
time.sleep(sleep_time)
|
| 68 |
+
except Exception as e:
|
| 69 |
+
logger.info(f"Failed to delete old version of nomic Atlas. Error: {e}")
|
| 70 |
+
|
| 71 |
|
| 72 |
|
| 73 |
def preprocess_markdown(text):
|