dcronin05/learn_mongodb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more testing

Browse Source
This commit is contained in:
Daniel Cronin 2024-10-05 18:28:58 -05:00
parent 974560e254
commit 8b8aefc403
2 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/extract.py
View File

@ -16,13 +16,13 @@ def insert_tag(r):
def insert(r):
global inserted
type = "document"
doc_type = "document"
pk = r['pk']
check = r['check']
content = r['content']
title = r['title']
lib.db.insert_one({'type': type,
lib.db.insert_one({'type': doc_type,
'title': title,
"content": content,
"checksum": check,
@ -103,7 +103,8 @@ def parse():
if 'title' in fields and 'content' in fields \
and doc['model'] == 'documents.document' \
and lib.sys.getsizeof(fields['content']) < 16777216:
and 50 < lib.sys.getsizeof(fields['content']) < 16777216\
and fields['content'] != "":
r['tags'] = doc['fields']['tags']
r['pk'] = doc['pk']
@ -113,9 +114,9 @@ def parse():
create_page(r)
if rec_exists(r):
continue
else:
insert(r)
f.close()

2
src/lib/__init__.py
View File

@ -10,7 +10,7 @@ def json_file():
def json_cursor(f):
items = ijson.items(f, 'item')
return (doc for doc in items )
return (doc for doc in items)
db = pymongo.MongoClient("10.0.0.59", 27017).paperless.content