more testing

src/bstack_api_calls.json

@@ -1,63 +0,0 @@
-{
-  'post_books_create',
-  'get_attachments_read',
-  'get_chapters_list',
-  'get_content_permissions_read',
-  'get_chapters_export_markdown',
-  'post_roles_create',
-  'put_recycle_bin_restore',
-  'put_image_gallery_update',
-  'put_books_update',
-  'post_shelves_create',
-  'get_books_read',
-  'get_attachments_list',
-  'put_content_permissions_update',
-  'get_recycle_bin_list',
-  'delete_users_delete',
-  'get_users_list',
-  'get_docs_json',
-  'get_books_export_html',
-  'put_chapters_update',
-  'get_chapters_export_pdf',
-  'get_pages_export_markdown',
-  'delete_pages_delete',
-  'post_attachments_create',
-  'put_attachments_update',
-  'get_roles_read',
-  'get_chapters_export_plain_text',
-  'get_users_read',
-  'delete_chapters_delete',
-  'put_pages_update',
-  'post_chapters_create',
-  'get_chapters_read',
-  'get_pages_list',
-  'get_pages_export_plain_text',
-  'get_roles_list',
-  'get_pages_export_html',
-  'get_image_gallery_read',
-  'delete_attachments_delete',
-  'post_users_create',
-  'get_audit_log_list',
-  'get_pages_export_pdf',
-  'delete_books_delete',
-  'get_shelves_read',
-  'delete_roles_delete',
-  'get_pages_read',
-  'put_shelves_update',
-  'get_books_export_plain_text',
-  'delete_shelves_delete',
-  'get_books_export_markdown',
-  'delete_image_gallery_delete',
-  'get_search_all',
-  'get_books_list',
-  'post_image_gallery_create',
-  'get_books_export_pdf',
-  'post_pages_create',
-  'get_shelves_list',
-  'get_docs_display',
-  'put_users_update',
-  'put_roles_update',
-  'delete_recycle_bin_destroy',
-  'get_chapters_export_html',
-  'get_image_gallery_list'
-}

src/bstack_api_calls.md

@@ -0,0 +1,121 @@
+`'post_books_create',`
+
+`'get_attachments_read',`
+
+`'get_chapters_list',`
+
+`'get_content_permissions_read',`
+
+`'get_chapters_export_markdown',`
+
+`'post_roles_create',`
+
+`'put_recycle_bin_restore',`
+
+`'put_image_gallery_update',`
+
+`'put_books_update',`
+
+`'post_shelves_create',`
+
+`'get_books_read',`
+
+`'get_attachments_list',`
+
+`'put_content_permissions_update',`
+
+`'get_recycle_bin_list',`
+
+`'delete_users_delete',`
+
+`'get_users_list',`
+
+`'get_docs_json',`
+
+`'get_books_export_html',`
+
+`'put_chapters_update',`
+
+`'get_chapters_export_pdf',`
+
+`'get_pages_export_markdown',`
+
+`'delete_pages_delete',`
+
+`'post_attachments_create',`
+
+`'put_attachments_update',`
+
+`'get_roles_read',`
+
+`'get_chapters_export_plain_text',`
+
+`'get_users_read',`
+
+`'delete_chapters_delete',`
+
+`'put_pages_update',`
+
+`'post_chapters_create',`
+
+`'get_chapters_read',`
+
+`'get_pages_list',`
+
+`'get_pages_export_plain_text',`
+
+`'get_roles_list',`
+
+`'get_pages_export_html',`
+
+`'get_image_gallery_read',`
+
+`'delete_attachments_delete',`
+
+`'post_users_create',`
+
+`'get_audit_log_list',`
+
+`'get_pages_export_pdf',`
+
+`'delete_books_delete',`
+
+`'get_shelves_read',`
+
+`'delete_roles_delete',`
+
+`'get_pages_read',`
+
+`'put_shelves_update',`
+
+`'get_books_export_plain_text',`
+
+`'delete_shelves_delete',`
+
+`'get_books_export_markdown',`
+
+`'delete_image_gallery_delete',`
+
+`'get_search_all',`
+
+`'get_books_list',`
+
+`'post_image_gallery_create',`
+
+`'get_books_export_pdf',`
+
+`'post_pages_create',`
+
+`'get_shelves_list',`
+
+`'get_docs_display',`
+
+`'put_users_update',`
+
+`'put_roles_update',`
+
+`'delete_recycle_bin_destroy',`
+
+`'get_chapters_export_html',`
+
+`'get_image_gallery_list'`

src/extract.py

@@ -3,28 +3,32 @@ from lib import json_cursor, json_file
 
 existing, inserted, duplicates = 0, 0, 0
 
+def insert_tag(r):
+    type = "tag"
+    pk = r['pk']
+    name = r['name']
+
+    lib.db.insert_one({
+        'type': type,
+        'pk': pk,
+        'name': name
+    })
+
 def insert(r):
     global inserted
+    doc_type = "document"
     pk = r['pk']
     check = r['check']
     content = r['content']
     title = r['title']
 
-    lib.db.insert_one({'title': title,"content": content,
+    lib.db.insert_one({'type': doc_type,
+                       'title': title,
+                       "content": content,
                        "checksum": check,
                        "pk": pk})
     inserted = inserted + 1
 
-def update_pk(r):
-    lib.db.update_one(
-        {
-            "checksum": r["check"]
-        },
-        {
-            "$set": {"index": r["index"]}
-        }
-    )
-
 def rec_exists(r):
     global existing, duplicates
     record = lib.db.find_one({"checksum": r["check"]})
@@ -38,6 +42,51 @@ def rec_exists(r):
         return True
     else: return False
 
+def create_page(r):
+    tags = r['tags']
+    pk = r['pk']
+    name = r['title']
+    markdown = r['content']
+    check = r['check']
+
+    t_list = []
+    for tag in tags:
+        t_list.append(
+            {"name": "pk", "value": str(tag)}
+        )
+    
+    mongo_tag = lib.db.find_one({"type": "tag", "pk": pk})
+    if mongo_tag:
+        chapter_id = lib.api.post_chapters_create({
+            'book_id': 3,
+            'name': mongo_tag['name'],
+            'tags': [{"name": "pk", "value": pk}]
+        })
+
+    try: chapter_id
+    except NameError: chapter_id = None
+
+    request = lib.api.post_pages_create({
+        'chapter_id': chapter_id['id'] if chapter_id else "",
+        'tags': t_list if len(tags) > 0 else [],
+        'book_id': 3,
+        'page_id': pk,
+        'name': name,
+        'markdown': markdown,
+    })
+
+    if 'error' in request:
+        print("\n\ndidn't work: ")
+        if 'validation' in request['error']:
+            print(request['error']['validation'])
+        else: print(str(request['error'])[0:1000])
+        print(t_list)
+        print(markdown[0:200])
+
+def tag_exists(t):
+    tag = lib.db.find_one({"pk": t['pk']})
+    return True if tag else False
+
 def parse():
     f = json_file()
     print("Parsing manifest json...")
@@ -45,13 +94,29 @@ def parse():
     # for every document in the export
     for doc in json_cursor(f):
         fields = doc['fields']
-        if 'title' in fields and 'content' in fields:
+        if doc['model'] == 'documents.tag':
+            tag = {'name': fields['name'], 'pk': doc['pk']}
+            if tag_exists(tag):
+                pass
+            else:
+                insert_tag(tag)
+            
+        if 'title' in fields and 'content' in fields \
+                and doc['model'] == 'documents.document' \
+                and 50 < lib.sys.getsizeof(fields['content']) < 16777216\
+                and fields['content'] != "":
+            
+            r['tags'] = doc['fields']['tags']
             r['pk'] = doc['pk']
             r['title'] = fields['title']
             r['content'] = fields['content']
             r['check'] = fields['checksum']
+
+            create_page(r)
+
             if rec_exists(r):
                 continue
             else:
                 insert(r)
+
     f.close()

src/lib/__init__.py

@@ -6,24 +6,18 @@ import bookstack
 
 
 def json_file():
-    return open("/mnt/tower/media/paperless/media/backup/test.json")
-
-# import paperless database export
-# manifest_path = "/mnt/user/media/paperless/media/manifest.json"
+    return open("/mnt/tower/media/paperless/media/backup/manifest.json")
 
 def json_cursor(f):
     items = ijson.items(f, 'item')
-    docs = (doc for doc in items \
-            if doc['model'] == "documents.document" \
-            if sys.getsizeof(doc['fields']['content']) < 16777216)
-    return docs
+    return (doc for doc in items)
 
 db = pymongo.MongoClient("10.0.0.59", 27017).paperless.content
 
 
 url = 'http://10.0.0.59:6875'
-token = 'RVSO8xZXOjRYJntNYPRd3E9iT2qXm11C'
-secret = 'qR5r2EyKT09ogz8VSolS12ispAV5QrT0'
+token = 'hd5H8OTYamMmKTh4LDamJrGDNk1MGdDk'
+secret = 'tUG7ZQIBCTczzZ5fqAeHj1Uh6rRzZIu9'
 
 api = bookstack.BookStack(url, token, secret)
 methods = api.generate_api_methods()