.gitignore

@@ -1,4 +1,3 @@
 .idea
 __pycache__
 *.code-workspace
-.venv*

src/bstack.py

@@ -1,9 +0,0 @@
-from lib import *
-
-
-def retrieve(id_num):
-    result = api.get_pages_read({"id": id_num})
-    print(result)
-
-def insert(id_num, name, text):
-    pass

src/bstack_api_calls.json

@@ -1,63 +0,0 @@
-{
-  'post_books_create',
-  'get_attachments_read',
-  'get_chapters_list',
-  'get_content_permissions_read',
-  'get_chapters_export_markdown',
-  'post_roles_create',
-  'put_recycle_bin_restore',
-  'put_image_gallery_update',
-  'put_books_update',
-  'post_shelves_create',
-  'get_books_read',
-  'get_attachments_list',
-  'put_content_permissions_update',
-  'get_recycle_bin_list',
-  'delete_users_delete',
-  'get_users_list',
-  'get_docs_json',
-  'get_books_export_html',
-  'put_chapters_update',
-  'get_chapters_export_pdf',
-  'get_pages_export_markdown',
-  'delete_pages_delete',
-  'post_attachments_create',
-  'put_attachments_update',
-  'get_roles_read',
-  'get_chapters_export_plain_text',
-  'get_users_read',
-  'delete_chapters_delete',
-  'put_pages_update',
-  'post_chapters_create',
-  'get_chapters_read',
-  'get_pages_list',
-  'get_pages_export_plain_text',
-  'get_roles_list',
-  'get_pages_export_html',
-  'get_image_gallery_read',
-  'delete_attachments_delete',
-  'post_users_create',
-  'get_audit_log_list',
-  'get_pages_export_pdf',
-  'delete_books_delete',
-  'get_shelves_read',
-  'delete_roles_delete',
-  'get_pages_read',
-  'put_shelves_update',
-  'get_books_export_plain_text',
-  'delete_shelves_delete',
-  'get_books_export_markdown',
-  'delete_image_gallery_delete',
-  'get_search_all',
-  'get_books_list',
-  'post_image_gallery_create',
-  'get_books_export_pdf',
-  'post_pages_create',
-  'get_shelves_list',
-  'get_docs_display',
-  'put_users_update',
-  'put_roles_update',
-  'delete_recycle_bin_destroy',
-  'get_chapters_export_html',
-  'get_image_gallery_list'
-}

src/extract.py

@@ -1,31 +1,20 @@
 import lib
-from lib import json_cursor, json_file
 
-existing, inserted, duplicates = 0, 0, 0
+# import paperless database export
+raw_manifest = open("/mnt/user/media/paperless/media/backup/manifest.json")
+manifest = lib.json.load(raw_manifest)
+
+index_num = 1
+
+existing, inserted, big, duplicates = 0, 0, 0, 0
 
 def insert(r):
-    global inserted
-    pk = r['pk']
-    check = r['check']
-    content = r['content']
-    title = r['title']
+    lib.db.insert_one({"title": r["title"],
+                               "content": r["content"],
+                               "checksum": r["check"],
+                               "index": r["index"]})
 
-    lib.db.insert_one({'title': title,"content": content,
-                       "checksum": check,
-                       "pk": pk})
-    inserted = inserted + 1
-
-def update_pk(r):
-    lib.db.update_one(
-        {
-            "checksum": r["check"]
-        },
-        {
-            "$set": {"index": r["index"]}
-        }
-    )
-
-def rec_exists(r):
+def exists(r):
     global existing, duplicates
     record = lib.db.find_one({"checksum": r["check"]})
     dupe = lib.db.find_one({"content": r["content"], "title": r["title"]})
@@ -39,19 +28,25 @@ def rec_exists(r):
     else: return False
 
 def parse():
-    f = json_file()
+    global inserted, big, index_num
     print("Parsing manifest json...")
-    r = {}
     # for every document in the export
-    for doc in json_cursor(f):
-        fields = doc['fields']
-        if 'title' in fields and 'content' in fields:
-            r['pk'] = doc['pk']
-            r['title'] = fields['title']
-            r['content'] = fields['content']
-            r['check'] = fields['checksum']
-            if rec_exists(r):
-                continue
-            else:
-                insert(r)
-    f.close()
+    for document in manifest:
+        #if the title and content tags aren't blank
+        try:
+            record = {"title": document["fields"]["title"],
+                      "content": document["fields"]["content"],
+                      "check": document["fields"]["checksum"],
+                      "index": index_num}
+
+            if lib.sys.getsizeof(record["content"]) < 16777216:
+                if record["content"] != "" and record["title"] != "":
+                    if not exists(record):
+                        insert(record)
+                        inserted = inserted + 1
+                        index_num = index_num + 1
+            else: big = big + 1
+        except KeyError:
+            continue
+
+

src/insert.py

@@ -0,0 +1,2 @@
+import lib
+

src/lib/__init__.py

@@ -1,29 +1,6 @@
 import pymongo
 import os
-import ijson
+import json
 import sys
-import bookstack
 
-
-def json_file():
-    return open("/mnt/tower/media/paperless/media/backup/test.json")
-
-# import paperless database export
-# manifest_path = "/mnt/user/media/paperless/media/manifest.json"
-
-def json_cursor(f):
-    items = ijson.items(f, 'item')
-    docs = (doc for doc in items \
-            if doc['model'] == "documents.document" \
-            if sys.getsizeof(doc['fields']['content']) < 16777216)
-    return docs
-
-db = pymongo.MongoClient("10.0.0.59", 27017).paperless.content
-
-
-url = 'http://10.0.0.59:6875'
-token = 'RVSO8xZXOjRYJntNYPRd3E9iT2qXm11C'
-secret = 'qR5r2EyKT09ogz8VSolS12ispAV5QrT0'
-
-api = bookstack.BookStack(url, token, secret)
-methods = api.generate_api_methods()
+db = pymongo.MongoClient("localhost", 27017).paperless.content

src/main.py

@@ -1,12 +1,13 @@
 import extract
 import search
+import insert
 
 def start():
     extract.parse()
     print("Existing:", extract.existing)
     print("Inserted:", extract.inserted)
+    print("To big:", extract.big)
     print("Dupes:", extract.duplicates)
-    extract.existing, extract.inserted, extract.duplicates = 0, 0, 0
 
     menu()
 
@@ -16,7 +17,8 @@ def prompt():
 def menu():
     value = prompt()
     if value == "reload":
-        start()
+        extract.parse()
+        value = prompt()
     while value != "quit" and value != "reload":
         results = search.query(value)
         if type(results) == dict: