changes and alterations HEAD main

1 files changed, 19 insertions, 7 deletions

diff --git a/4_delete.py b/4_delete.py
index 615fbab..0e47e52 100644
--- a/4_delete.py
+++ b/4_delete.py
@@ -3,9 +3,10 @@ import time
 from pathlib import Path
 
 import httpx
+import psutil
 import psycopg
 
-from com import eval_config, parse_graph, progressbar, FilterAction
+from com import FilterAction, eval_config, parse_graph, progressbar
 
 config = eval_config()
 conn: psycopg.Connection = config["connect"]()
@@ -94,17 +95,25 @@ for note, action in queue:
             pb.update(message="down")
             time.sleep(1)
             continue
+        Path('queue-stats.dump').write_text(f"status:{resp.status_code}\nbody:\n{resp.text}")
         deliver_waiting = resp.json()["deliver"]["waiting"]
         obliterate_waiting = resp.json()["obliterate"]["waiting"]
-        obliterate_delayed = resp.json()["obliterate"]["delayed"]
-        if deliver_waiting < 100 and obliterate_waiting + obliterate_delayed< 50000:
+        if deliver_waiting < 100 and obliterate_waiting < 50000:
             break
-        pb.update(message=f"queue ({deliver_waiting}/{obliterate_waiting + obliterate_delayed})")
+        pb.update(message=f"queue ({deliver_waiting}/{obliterate_waiting})")
+        time.sleep(10)
+
+    # make sure there's enough memory for new jobs
+    while True:
+        vmem = psutil.virtual_memory()
+        if vmem.available > (512 * 1024 * 1024):
+            break
+        pb.update(message="memory")
         time.sleep(10)
 
     # prevent api rate limiting
     req_delay = time.time() - last_req
-    if req_delay < 15:
+    if req_delay < 30:
         pb.update(message="delaying")
         time.sleep(req_delay)
 
@@ -122,16 +131,19 @@ for note, action in queue:
             continue
         elif resp.status_code == 502:
             pb.update(status="down")
-            continue
             time.sleep(1)
+            continue
         elif resp.status_code >= 400:
             body = resp.json()
             if body["error"]["code"] == "NO_SUCH_NOTE":
                 pb.increment(message="seeking")
                 seeking = True
                 break
+            elif body["error"]["code"] == "QUEUE_FULL":
+                print("\nobliterate queue overflowed, exiting to save server")
+                break
             err += 1
-            if err > 10:
+            if err > 3:
                 raise Exception(f"{body['error']['code']}: {body['error']['message']}")
             sys.stdout.write("\r")
             print(f"err {body['error']['code']} {body['error']['message']} ")