working on tag generation

src/extract.py

@@ -3,28 +3,32 @@ from lib import json_cursor, json_file
 
 existing, inserted, duplicates = 0, 0, 0
 
+def insert_tag(r):
+    type = "tag"
+    pk = r['pk']
+    name = r['name']
+
+    lib.db.insert_one({
+        'type': type,
+        'pk': pk,
+        'name': name
+    })
+
 def insert(r):
     global inserted
+    type = "document"
     pk = r['pk']
     check = r['check']
     content = r['content']
     title = r['title']
 
-    lib.db.insert_one({'title': title,"content": content,
+    lib.db.insert_one({'type': type, 
+                       'title': title,
+                       "content": content,
                        "checksum": check,
                        "pk": pk})
     inserted = inserted + 1
 
-def update_pk(r):
-    lib.db.update_one(
-        {
-            "checksum": r["check"]
-        },
-        {
-            "$set": {"index": r["index"]}
-        }
-    )
-
 def rec_exists(r):
     global existing, duplicates
     record = lib.db.find_one({"checksum": r["check"]})
@@ -45,6 +49,7 @@ def create_page(r):
     check = r['check']
 
     request = lib.api.post_pages_create({
+        'tags': 
         'book_id': 3,
         'page_id': pk,
         'name': name,
@@ -54,6 +59,10 @@ def create_page(r):
     # if 'message' in request:
     print(request)
 
+def tag_exists(t):
+    tag = lib.db.find_one({"pk": t['pk']})
+    return True if tag else False
+
 def parse():
     f = json_file()
     print("Parsing manifest json...")
@@ -61,7 +70,18 @@ def parse():
     # for every document in the export
     for doc in json_cursor(f):
         fields = doc['fields']
-        if 'title' in fields and 'content' in fields:
+        if doc['model'] == 'documents.tag':
+            tag = {'name': fields['name'], 'pk': doc['pk']}
+            if tag_exists(tag):
+                pass
+            else:
+                insert_tag(tag)
+            
+        if 'title' in fields and 'content' in fields \
+                and doc['model'] == 'documents.document' \
+                and lib.sys.getsizeof(fields['content']) < 16777216:
+            
+            r['tags'] = doc['fields']['tags']
             r['pk'] = doc['pk']
             r['title'] = fields['title']
             r['content'] = fields['content']
@@ -69,6 +89,7 @@ def parse():
 
             create_page(r)
 
+
             if rec_exists(r):
                 continue
             else:

src/lib/__init__.py

@@ -6,17 +6,11 @@ import bookstack
 
 
 def json_file():
-    return open("/mnt/tower/media/paperless/media/backup/test.json")
-
-# import paperless database export
-# manifest_path = "/mnt/user/media/paperless/media/manifest.json"
+    return open("/mnt/tower/media/paperless/media/backup/manifest.json")
 
 def json_cursor(f):
     items = ijson.items(f, 'item')
-    docs = (doc for doc in items \
-            if doc['model'] == "documents.document" \
-            if sys.getsizeof(doc['fields']['content']) < 16777216)
-    return docs
+    return (doc for doc in items )
 
 db = pymongo.MongoClient("10.0.0.59", 27017).paperless.content