Lynch/processorupdate (#122)

learning_observer/learning_observer/doc_processor.py

@@ -31,6 +31,8 @@
 import writing_observer.languagetool
 import writing_observer.writing_analysis
 
+from learning_observer.log_event import debug_log
+
 pmss.register_field(
     name='document_processing_delay_seconds',
     type=pmss.pmsstypes.TYPES.integer,
@@ -255,19 +257,22 @@ async def check_rules(rules, doc_id):
 async def process_document(doc_id):
     if not await check_rules(RULES, doc_id):
         return False
-    print('* Starting to process document:', doc_id)
+    debug_log('* Processing document:', doc_id)
     doc_text = None
     student_id = await _determine_student(doc_id)
     google_auth = await _fetch_teacher_credentials(student_id)
     # TODO this could be cleaned up
     if google_auth is not None:
-        doc_text = await _fetch_doc_text_from_google(doc_id, google_auth)
+        doc_text = await _fetch_doc_text_from_google(doc_id, google_auth, student_id)
     if doc_text is None or len(doc_text) == 0:
         doc_text = await _fetch_doc_text_from_reconstruct(doc_id, student_id)
         if doc_text is None or len(doc_text) == 0:
             failed_fetch.add(doc_id)
             return False
     await _pass_doc_through_analysis(doc_id, doc_text, student_id)
+
+    debug_log('**  Done processing document:', doc_id)
+
     return True
 
 
@@ -317,14 +322,22 @@ async def _fetch_doc_text_from_reconstruct(doc_id, student_id):
     return reconstruct['text']
 
 
-async def _fetch_doc_text_from_google(doc_id, creds):
+async def _fetch_doc_text_from_google(doc_id, creds, student_id):
     '''Fetch the document text from the appropriate
     Google endpoint.
     '''
     runtime = fetch_mock_runtime(creds)
     response = await learning_observer.google.doctext(runtime, documentId=doc_id)
     if 'text' not in response:
         return None
+
+    key = sa_helpers.make_key(
+        writing_observer.writing_analysis.reconstruct,
+        {sa_helpers.EventField('doc_id'): doc_id, sa_helpers.KeyField.STUDENT: student_id},
+        sa_helpers.KeyStateType.INTERNAL
+    )
+    await KVS.set(key, response)
+
     return response['text']
 
 
@@ -335,9 +348,11 @@ async def _pass_doc_through_analysis(doc_id, text, student_id):
     - Running through Language Tool
     Then store the documents in the default KVS
     '''
+    # Can log above and below these lines for primary processing work.
     awe_output = writing_observer.awe_nlp.process_text(text)
     prep_for_lt = [{'text': text, 'provenance': None}]
     lt_output = await writing_observer.languagetool.process_texts(prep_for_lt)
+
     output = {
         'document_id': doc_id,
         'student_id': student_id,
@@ -372,6 +387,7 @@ async def _pass_doc_through_analysis(doc_id, text, student_id):
         {sa_helpers.EventField('doc_id'): doc_id, sa_helpers.KeyField.STUDENT: student_id},
         sa_helpers.KeyStateType.INTERNAL
     )
+
     await KVS.set(key, output)
 
 

servermanagement/RunLearningObserver.sh

@@ -1,4 +1,3 @@
-
 #!/usr/bin/env bash
 # ===============================
 # RunLearningObserver.sh
@@ -13,23 +12,32 @@
 
 # System Variables
 # --------------------------------------
-VIRTUALENV_PATH="/usr/local/share/projects/WritingObserver/VirtualENVs/WOvenv"
+VIRTUALENV_PATH="/usr/local/share/projects/WritingObserver/VENV/WOVenv"
 #VIRTUALENV_PYTHON="/usr/local/share/Projects/WritingObserver/VirtualENVs/learning_observer/bin/python3.9"
 LEARNING_OBSERVER_LOC="/usr/local/share/projects/WritingObserver/Repositories/ArgLab_writing_observer/learning_observer"
 LOGFILE_DEST="/usr/local/share/projects/WritingObserver/Repositories/ArgLab_writing_observer/learning_observer/learning_observer/logs"
 
+
 # Make the logfile name
 # ---------------------------------------
 LOG_DATE=$(date "+%m-%d-%Y--%H-%M-%S")
 LOGFILE_NAME="$LOGFILE_DEST/learning_observer_service_$LOG_DATE.log"
 echo $LOG_NAME;
 
-
+DOC_PROCESSOR_LOG="$LOGFILE_DEST/document_processor_service_$LOG_DATE.log"
+echo $DOC_PROCESSOR_LOG;
+
 # Now run the thing.
 # --------------------------------------
 echo "Running Learning Observer Service..."
 cd $LEARNING_OBSERVER_LOC
+
 source $VIRTUALENV_PATH/bin/activate
+
+nohup python learning_observer/doc_processor.py > $DOC_PROCESSOR_LOG 2>&1 &
+DOC_PROCESS_ID=$!
+echo $DOC_PROCESS_ID > $LOGFILE_DEST/doc_run.pid
+
 nohup python learning_observer > $LOGFILE_NAME 2>&1 &
 PROCESS_ID=$!
 echo $PROCESS_ID > $LOGFILE_DEST/run.pid