dcronin05/learn_mongodb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more testing

Browse Source
This commit is contained in:
Daniel Cronin 2024-10-05 18:28:58 -05:00
parent 974560e254
commit 8b8aefc403
2 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/extract.py
View File

@ -16,13 +16,13 @@ def insert_tag(r):
def insert(r): def insert(r):
global inserted global inserted
type = "document" doc_type = "document"
pk = r['pk'] pk = r['pk']
check = r['check'] check = r['check']
content = r['content'] content = r['content']
title = r['title'] title = r['title']
lib.db.insert_one({'type': type, lib.db.insert_one({'type': doc_type,
'title': title, 'title': title,
"content": content, "content": content,
"checksum": check, "checksum": check,
@ -103,7 +103,8 @@ def parse():
if 'title' in fields and 'content' in fields \ if 'title' in fields and 'content' in fields \
and doc['model'] == 'documents.document' \ and doc['model'] == 'documents.document' \
and lib.sys.getsizeof(fields['content']) < 16777216: and 50 < lib.sys.getsizeof(fields['content']) < 16777216\
and fields['content'] != "":
r['tags'] = doc['fields']['tags'] r['tags'] = doc['fields']['tags']
r['pk'] = doc['pk'] r['pk'] = doc['pk']
@ -113,9 +114,9 @@ def parse():
create_page(r) create_page(r)
if rec_exists(r): if rec_exists(r):
continue continue
else: else:
insert(r) insert(r)
f.close() f.close()

2
src/lib/__init__.py
View File

@ -10,7 +10,7 @@ def json_file():
def json_cursor(f): def json_cursor(f):
items = ijson.items(f, 'item') items = ijson.items(f, 'item')
return (doc for doc in items ) return (doc for doc in items)
db = pymongo.MongoClient("10.0.0.59", 27017).paperless.content db = pymongo.MongoClient("10.0.0.59", 27017).paperless.content