updated code to use less logins to database

code/translate.py

@@ -8,7 +8,6 @@ from tqdm import tqdm
 import logging
 
 from database import Data_table
-from database import insert_translated
 from clean_data import clean_content
 from setup_logging import setup_logging
 
@@ -27,15 +26,14 @@ def translate(text, target_lang='en'):
         return ''
     return text_trans
 
-def get_untranslated():
+def get_untranslated(dt):
-    with Data_table() as dt:
+    query = (
-        query = (
+        f'SELECT * FROM {dt._name} '
-            f'SELECT * FROM {dt._name} '
+        'WHERE content_en IS NULL '
-            'WHERE content_en IS NULL '
+        'ORDER BY random() '
-            'ORDER BY random() '
+        'LIMIT 1 '
-            'LIMIT 1 '
+    )
-        )
+    df = pd.read_sql_query(query, dt.con)
-        df = pd.read_sql_query(query, dt.con)
     if len(df)==0: return None
     val_dict = {
         'id': int(df['id'][0]),
@@ -43,36 +41,47 @@ def get_untranslated():
     }
     return val_dict
 
+def insert_translated(dt, val_dict):
+    assert 'id' in val_dict.keys()
+    assert 'content_en' in val_dict.keys()
+    query = (
+        f'UPDATE {dt._name} ' 
+        ' SET content_en = (%(content_en)s) '
+        'WHERE id = (%(id)s) '
+    )
+    dt.execute(query, val_dict)
+
 def continously_translate():
-    logging.info('began translating')
+    with Data_table() as dt:
-    while True:
+        logging.info('began translating')
-        try:
+        while True:
-            val_dict = get_untranslated()
+            try:
-        except Exception as e:
+                val_dict = get_untranslated(dt)
-            logging.warning(f'error when getting untranslated content from database: {e}')
+            except Exception as e:
-            continue
+                logging.warning(f'error when getting untranslated content from database: {e}')
-        if val_dict is None: break
+                continue
-        # clean
+            if val_dict is None: break
-        try:
+            # clean
-            content_clean = clean_content(val_dict['content'])
+            try:
-        except Exception as e:
+                content_clean = clean_content(val_dict['content'])
-            logging.warning(f'error when cleaning content: {e}')
+            except Exception as e:
-            continue
+                logging.warning(f'error when cleaning content: {e}')
-        # translate
+                continue
-        try:
+            # translate
-            content_en = translate(content_clean)
+            try:
-        except Exception as e:
+                content_en = translate(content_clean)
-            logging.warning(f'error when translating content: {e}')
+            except Exception as e:
-            continue
+                logging.warning(f'error when translating content: {e}')
-        # upload to database
+                continue
-        val_dict['content_en'] = content_en
+            # upload to database
-        try:
+            val_dict['content_en'] = content_en
-            insert_translated(val_dict)
+            try:
-        except Exception as e:
+                insert_translated(dt, val_dict)
-            logging.warning(f'error when sending translated content to database: {e}')
+            except Exception as e:
-            continue
+                logging.warning(f'error when sending translated content to database: {e}')
-        logging.info('translated content')
+                continue
-    logging.info('no more untranslated content located')
+            logging.info('translated content')
+        logging.warning('stopped translating')
 
 if __name__ == '__main__':
     load_dotenv()