more testing

src/extract.py

@@ -16,13 +16,13 @@ def insert_tag(r):
 
 def insert(r):
     global inserted
-    type = "document"
+    doc_type = "document"
     pk = r['pk']
     check = r['check']
     content = r['content']
     title = r['title']
 
-    lib.db.insert_one({'type': type, 
+    lib.db.insert_one({'type': doc_type,
                        'title': title,
                        "content": content,
                        "checksum": check,
@@ -103,7 +103,8 @@ def parse():
             
         if 'title' in fields and 'content' in fields \
                 and doc['model'] == 'documents.document' \
-                and lib.sys.getsizeof(fields['content']) < 16777216:
+                and 50 < lib.sys.getsizeof(fields['content']) < 16777216\
+                and fields['content'] != "":
             
             r['tags'] = doc['fields']['tags']
             r['pk'] = doc['pk']
@@ -113,9 +114,9 @@ def parse():
 
             create_page(r)
 
-
             if rec_exists(r):
                 continue
             else:
                 insert(r)
+
     f.close()

src/lib/__init__.py

@@ -10,14 +10,14 @@ def json_file():
 
 def json_cursor(f):
     items = ijson.items(f, 'item')
-    return (doc for doc in items )
+    return (doc for doc in items)
 
 db = pymongo.MongoClient("10.0.0.59", 27017).paperless.content
 
 
 url = 'http://10.0.0.59:6875'
-token = 'RVSO8xZXOjRYJntNYPRd3E9iT2qXm11C'
-secret = 'qR5r2EyKT09ogz8VSolS12ispAV5QrT0'
+token = 'hd5H8OTYamMmKTh4LDamJrGDNk1MGdDk'
+secret = 'tUG7ZQIBCTczzZ5fqAeHj1Uh6rRzZIu9'
 
 api = bookstack.BookStack(url, token, secret)
 methods = api.generate_api_methods()