Change argument type to string, parser interface to use path

- All of this is to address issue #9
- support input json (sort of, needs testing)

bin/smstools

@@ -7,8 +7,8 @@ import smstools
 
 term = smstools.term
 parser = argparse.ArgumentParser(description='Import texts to android sms database file.')
-parser.add_argument('infiles', nargs='+', type=argparse.FileType('r'), help='input files, may include multiple sources')
-parser.add_argument('outfile', type=argparse.FileType('w'), help='output file to write to.')
+parser.add_argument('infiles', nargs='+', type=str, help='input files, may include multiple sources')
+parser.add_argument('outfile', type=str, help='output file to write to.')
 parser.add_argument('--type', type=str, help='output type', choices=smstools.OUTPUT_TYPE_CHOICES.keys())
 parser.add_argument('-V', '--version', action='version', version="%(prog)s "+smstools.getVersion())
 
@@ -21,25 +21,34 @@ except:
 args = parser.parse_args()
 
 try:
+
+    ## Check arguments ##
+
+    for filepath in args.infiles:
+        if not os.path.exists(filepath) or os.path.getsize(filepath) < 1:
+            raise ArgumentError("%s doesn't exist or has no data" % filepath)
+    if os.path.exists(args.outfile):
+        raise ArgumentError("output file [%s] can't exist" % args.outfile)
+
     outtype = args.type
     if outtype:
         outtype = smstools.OUTPUT_TYPE_CHOICES[outtype]
     else:
-        extension = os.path.splitext(args.outfile.name)[1]
+        extension = os.path.splitext(args.outfile)[1]
         if extension in smstools.EXTENTION_TYPE_DEFAULTS:
             outtype = smstools.OUTPUT_TYPE_CHOICES[smstools.EXTENTION_TYPE_DEFAULTS[extension]]
         else:
             raise smstools.SMSToolError("unknown output format (use --type argument)")
 
-    #get the texts into memory
+
+    ## get the texts into memory ##
     texts = []
-    for file in args.infiles:
+    for filename in args.infiles:
         starttime = time.time() #meause execution time
-        parser = smstools.getParser(file)
-        if os.path.splitext(file.name)[1] == ".db": file.close()
-        new_texts = parser.parse(file.name)
+        parser = smstools.getParser(filename)
+        new_texts = parser.parse(filename)
         print term.black_on_blue("%d messages read in %d seconds from %s using parser %s") % \
-            (len(new_texts), (time.time()-starttime), os.path.basename(file.name), parser.__class__)
+            (len(new_texts), (time.time()-starttime), os.path.basename(filename), parser.__class__)
 
         if len(new_texts):
             print "  latest text: " + term.blue(new_texts[-1].localStringTime()) + " to " + term.blue(new_texts[-1].num) + ":"
@@ -50,7 +59,7 @@ try:
     texts = sorted(texts, key=lambda text: text.date)
 
     outtype().write(texts, args.outfile)
-    print term.black_on_yellow("saved messages to %s") % os.path.basename(args.outfile.name)
+    print term.black_on_yellow("saved messages to %s") % os.path.basename(args.outfile)
 except smstools.SMSToolError as e:
     sys.exit(e) #exit with error without traceback
 except KeyboardInterrupt:

setup.py

@@ -21,7 +21,7 @@ def read(fname):
     license='CC BY-NC-SA 3.0 US',
     install_requires=['unicodecsv>=0.9.3'],
     extras_require = {
-        'colors':  ["blessings>=1.5.0"]
-        'autocomplete':  ["argcomplete>=0.8.0"]
+        'colors':  ["blessings>=1.5.0"],
+        'autocomplete':  ["argcomplete>=0.8.0"],
     },
 )

smstools/android.py

@@ -5,10 +5,10 @@ class Android:
     """ Android sqlite reader and writer """
 
 
-    def parse(self, file):
+    def parse(self, filepath):
         """ Parse a sqlite file to Text[] """
 
-        db = sqlite3.connect(file)
+        db = sqlite3.connect(filepath)
         cursor = db.cursor()
         texts = self.parse_cursor(cursor)
         cursor.close()
@@ -26,21 +26,11 @@ def parse_cursor(self, cursor):
             texts.append(txt)
         return texts
 
-    def write(self, texts, outfile):
+    def write(self, texts, outfilepath):
         """ write a Text[] to sqlite file """
-        if type(outfile) == file:
-            if not file.closed:
-                file.close()
-            outfile = os.path.abspath(outfile.name)
-
-        if (os.path.isfile(outfile) and (os.path.getsize(outfile) > 0)):
-            print "connecting to existing db"
-            conn = sqlite3.connect(outfile)
-        else:
-            print "Creating empty Android SQLITE db"
-            conn = sqlite3.connect(outfile)
-            conn.executescript(INIT_DB_SQL)
-
+        print "Creating empty Android SQLITE db"
+        conn = sqlite3.connect(outfile)
+        conn.executescript(INIT_DB_SQL)
         cursor = conn.cursor()
 
         self.write_cursor(texts, cursor)

smstools/core.py

@@ -38,16 +38,17 @@ def __eq__(self, other):
         return self.__dict__ == other.__dict__
 
 
-def getParser(file):
-    extension = os.path.splitext(file.name)[1]
+def getParser(filepath):
+    extension = os.path.splitext(filepath)[1]
     if extension == ".csv":
         return csv.CSV()
+    if extension == ".json":
+        return jsoner.JSONer()
     elif extension == ".db" or extension == ".sqlite":
-        file.close()
         try:
-            tableNames = core.getDbTableNames( file.name )
+            tableNames = core.getDbTableNames( filepath )
         except:
-            raise UnrecognizedDBError("Error reading from %s" % file.name)
+            raise UnrecognizedDBError("Error reading from %s" % filepath)
         if "handle" in tableNames:
             return ios6.IOS6()
         elif "group_member" in tableNames:
@@ -56,8 +57,9 @@ def getParser(file):
             return googlevoice.GoogleVoice()
         elif "sms" in tableNames:
             return android.Android()
-        info = getDbInfo( file.name )
-        raise UnrecognizedDBError("Unknown sqlite database: [%s]\n%s" % (os.path.basename(file.name), info))
+        print term.red_on_black("unrecognized database details and structure:")
+        print getDbInfo( file.name )
+        raise UnrecognizedDBError("Unknown sqlite database: [%s]" % os.path.basename(filepath))
     elif extension == ".xml":
         return xmlmms.XMLmms()
     raise UnrecognizedDBError("Unknown extension %s" % extension)

smstools/googlevoice.py

@@ -4,14 +4,14 @@
 class GoogleVoice:
     """ Google Voice (in sqlite or format) reader and writer """
 
-    def parse(self, file):
-        return self.parseSQL(file)
+    def parse(self, filepath):
+        return self.parseSQL(filepath)
         #TODO add direct files parsing?
 
-    def parseSQL(self, file):
+    def parseSQL(self, filepath):
         """ Parse a GV sqlite file to Text[] """
 
-        conn = sqlite3.connect(file)
+        conn = sqlite3.connect(filepath)
         c = conn.cursor()
         texts = []
         query = c.execute(
@@ -29,6 +29,6 @@ def parseSQL(self, file):
             texts.append(txt)
         return texts
 
-    def write(self, texts, outfile):
+    def write(self, texts, outfilepath):
         raise Exception("Can't output to Google Voice, sorry.")
 

smstools/ios5.py

@@ -6,9 +6,9 @@ class IOS5:
     """ iOS 5 sqlite reader and writer """
 
 
-    def parse(self, file):
+    def parse(self, filepath):
         """ Parse iOS 5 sqlite file to Text[] """
-        db = sqlite3.connect(file)
+        db = sqlite3.connect(filepath)
         cursor = db.cursor()
         texts = self.parse_cursor(cursor)
         cursor.close()

smstools/ios6.py

@@ -5,9 +5,9 @@
 class IOS6:
     """ iOS 6 sqlite reader and writer """
 
-    def parse(self, file):
+    def parse(self, filepath):
         """ Parse iOS 6 sqlite file to Text[] """
-        db = sqlite3.connect(file)
+        db = sqlite3.connect(filepath)
         cursor = db.cursor()
         texts = self.parse_cursor(cursor)
         cursor.close()
@@ -31,19 +31,10 @@ def parse_cursor(self, cursor):
             texts.append(text)
         return texts
 
-    def write(self, texts, outfile):
-        if type(outfile) == file:
-            if not file.closed:
-                file.close()
-            outfile = os.path.abspath(outfile.name)
-
-        if (os.path.isfile(outfile) and (os.path.getsize(outfile) > 0)):
-            print "connecting to existing db"
-            conn = sqlite3.connect(outfile)
-        else:
-            print "Creating empty Android SQLITE db"
-            conn = sqlite3.connect(outfile)
-            conn.executescript(INIT_DB_SQL)
+    def write(self, texts, outfilepath):
+        print "Creating empty iOS 6 SQLITE db"
+        conn = sqlite3.connect(outfilepath)
+        conn.executescript(INIT_DB_SQL)
 
         cursor = conn.cursor()
         self.write_cursor(texts, cursor)
@@ -64,36 +55,38 @@ def write_cursor(self, texts, cursor):
         chat_lookup = {} # chat_key -> chat ROWID
         chat_participants = {} # chat_key -> [cleaned1, cleaned2]
         for txt in texts:
-            clean_number = core.cleanNumber(txt.num)
-            chat_key = txt.chatroom if txt.chatroom else txt.num
+            try:
+                clean_number = core.cleanNumber(txt.num)
+                chat_key = txt.chatroom if txt.chatroom else txt.num
 
-            ## Create the handle table (effectively a contacts table)
-            if not clean_number in handles_lookup:
-                try:
+                ## Create the handle table (effectively a contacts table)
+                if (clean_number) and (not clean_number in handles_lookup):
                     cursor.execute( "INSERT INTO handle ('id', service, uncanonicalized_id ) \
                         VALUES (?,?,?)", [txt.num,"SMS",clean_number])
-                except:
-                    print "failed at: %s" % (txt)
-                    raise
-                handles_lookup[clean_number] = cursor.lastrowid
-
-            ## Create the chat table (effectively a threads table)
-            if not chat_key in chat_lookup:
-                guid = ("SMS;+;%s" % txt.chatroom) if txt.chatroom else ("SMS;-;%s" % txt.num)
-                style = 43 if txt.chatroom else 45
-                cursor.execute( "INSERT INTO chat (guid, style, state, chat_identifier, service_name, room_name ) \
-                    VALUES (?,?,?,?,?,?)", [guid, style, 3, chat_key, 'SMS', txt.chatroom])
-                chat_lookup[chat_key] = cursor.lastrowid
-
-            ## Create the chat_handle_join table (represents participants in all threads)
-            if not chat_key in chat_participants:
-                chat_participants[chat_key] = set()
-            if not clean_number in chat_participants[chat_key]:
-                chat_participants[chat_key].add(clean_number)
-                chat_id = chat_lookup[chat_key]
-                handle_id = handles_lookup[clean_number]
-                cursor.execute( "INSERT INTO chat_handle_join (chat_id, handle_id ) \
-                    VALUES (?,?)", [chat_id, handle_id])
+                    handles_lookup[clean_number] = cursor.lastrowid
+
+                if not chat_key:
+                    core.warning("no txt chat_key for " + txt)
+                ## Create the chat table (effectively a threads table)
+                if not chat_key in chat_lookup:
+                    guid = ("SMS;+;%s" % txt.chatroom) if txt.chatroom else ("SMS;-;%s" % txt.num)
+                    style = 43 if txt.chatroom else 45
+                    cursor.execute( "INSERT INTO chat (guid, style, state, chat_identifier, service_name, room_name ) \
+                        VALUES (?,?,?,?,?,?)", [guid, style, 3, chat_key, 'SMS', txt.chatroom])
+                    chat_lookup[chat_key] = cursor.lastrowid
+
+                ## Create the chat_handle_join table (represents participants in all threads)
+                if not chat_key in chat_participants:
+                    chat_participants[chat_key] = set()
+                if not clean_number in chat_participants[chat_key]:
+                    chat_participants[chat_key].add(clean_number)
+                    chat_id = chat_lookup[chat_key]
+                    handle_id = handles_lookup[clean_number]
+                    cursor.execute( "INSERT INTO chat_handle_join (chat_id, handle_id ) \
+                        VALUES (?,?)", [chat_id, handle_id])
+            except:
+                print core.term.red("something failed at: %s") % (txt)
+                raise
 
 
         print "built handles table with %i, chat with %i, chat_handle_join with %i entries" \

smstools/jsoner.py

@@ -5,8 +5,12 @@
 class JSONer:
     """ Json reader and writer """
 
-    def parse(self, file):
-        return self.fromJson(file.read())
+    def parse(self, filepath):
+        file = open(filepath, 'r')
+        try:
+            return self.fromJson(file.read())
+        finally:
+            file.close()
 
     def fromJson(self, string):
         def asTexts(dct):
@@ -18,6 +22,10 @@ def asTexts(dct):
     def toJson(self, texts):
         return json.dumps(texts, default=lambda o: o.__dict__, sort_keys=True, indent=4)
 
-    def write(self, texts, outfile):
-        outfile.write(self.toJson(texts))
+    def write(self, texts, outfilepath):
+        outfile = open(outfilepath, 'w')
+        try:
+            outfile.write(self.toJson(texts))
+        finally:
+            outfile.close()
 

smstools/sms_exceptions.py

@@ -13,3 +13,6 @@ class UnfinishedError(SMSToolError):
 
 class NonEmptyStartDBError(SMSToolError):
     pass
+
+class ArgumentError(SMSToolError):
+    pass

smstools/tabular.py

@@ -7,8 +7,9 @@ class Tabular:
     """ Google Voice (in sqlite or format) reader and writer """
 
 
-    def parse(self, file):
+    def parse(self, filepath):
         """ Parse a GV CSV file to Text[] """
+        file = open(filepath, 'r')
         inreader = unicodecsv.reader(file, encoding='utf-8')
 
         #gather needed column indexes from the csv file
@@ -34,12 +35,15 @@ def parse(self, file):
                     row[bodyIndex] ) #body
             texts.append(txt)
             i += 1
+        file.close()
         return texts
 
-    def write(self, texts, outfile):
+    def write(self, texts, outfilepath):
+        outfile = open(outfilepath, 'w')
         writer = unicodecsv.writer(outfile, quoting=unicodecsv.QUOTE_NONNUMERIC, encoding='utf-8')
         writer.writerow(texts[0].__dict__.keys())
         writer.writerows( [text.__dict__.values() for text in texts] )
+        outfile.close()
 
 
 

smstools/tests/android_sqlite_tests.py

@@ -21,6 +21,7 @@ def test_write_parse(self):
 
         for i in range(len(true_texts)):
             self.assertEqual(true_texts[i], parsed_texts[i])
+        cursor.close()
 
 
 if __name__ == '__main__':

smstools/tests/ios6_tests.py

@@ -22,12 +22,6 @@ def test_write_parse(self):
         for i in range(len(true_texts)):
             self.assertTextsEqual(true_texts[i], parsed_texts[i])
 
-    def test_write_parse(self):
-        t = 430282537
-
-
-        print time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(1347517119))
-
 
     def assertTextsEqual(self, t1, t2):
         warns = core.compareTexts(t1, t2,

smstools/xmlmms.py

@@ -5,18 +5,18 @@ class XMLmms:
     """ Android XML reader and writer """
 
 
-    def parse(self, file):
+    def parse(self, filepath):
         """ Parse XML file to Text[] """
         texts = []
-        dom = xml.dom.minidom.parse(file)
+        dom = xml.dom.minidom.parse(open(filepath,'r'))
         i = 0
         for sms in dom.getElementsByTagName("sms"):
             txt = core.Text( sms.attributes['address'].value, sms.attributes['date'].value,
                     (sms.attributes['type'].value==2), sms.attributes['body'].value)
             texts.append(txt)
         return texts
 
-    def write(self, texts, outfile):
+    def write(self, texts, outfilepath):
         """ write a Text[] to XML file """
         doc = xml.dom.minidom.Document()
         doc.encoding = "UTF-8"
@@ -39,4 +39,6 @@ def write(self, texts, outfile):
             # print doc.toprettyxml(indent="  ", encoding="UTF-8")
         print "generating xml output"
         xmlout = doc.toprettyxml(indent="  ", encoding="UTF-8")
+        outfile = open(outfilepath,'w')
         outfile.write(xmlout)
+        outfile.close()