This is deprecated -- these days I parse the metadata files directly in Local parsing of KOReader Notes to Org Roam
This is a Literate Programming notebook which extracts the notes and presents a list of them. KOReader has an "export notes to JSON file" function in the Evernote export plugin menu. That puts a file in ~/koreader/clipboard on the tablet.
These notes are viewable/greppable in ./remarkable_notes/ and should only be refered to in other pages in the Archive for now so that interlinking and summarization and other Wiki Gardening can be done without clobbering the files or polluting the org-roam database with auto-generated IDs.
The files are in "=remarkable_notes=" because these were originally made for my reMarkable Tablet, though now I use KOReader on a Boox Onyx android tablet with KOReader installed from a side-loaded F-droid. What a pain, what a joyous pain... lol. KOreader on android crashes in the highlight export program, so I bundle a patched version of KOReader in
I navigate in KOReader =Tools=-> =Highlights=-> Export all notes in your library, then with the tablet plugged in and not "sleeping", I get a list of the files by hitting C-c C-c on the CALL line below:
Org Mode Hypermedia generates the koreader note index dynamically
Another attempt at JSON-parsing
the lua table format was not really structured to be parsed and anyways fennel/lua kind of made working with the text a pain.
import argparse
parser = argparse.ArgumentParser(
prog='koreader-to-org',
description='convert koreader clippings to org-mode nodes',
epilog='hey smell this')
parser.add_argument('filename')
parser.add_argument('-d', '--dir', default='~/org/highlights')
parser.add_argument('-f', '--force', action='store_true')
parser.add_argument('-n', '--dry-run', action='store_true')
parser.add_argument('-v', '--verbose',
action='store_true')
args = parser.parse_args()
print(args.filename, args.verbose)
def collect_chapters(document):
time = 0
chaps = dict() # (ab)use dict as ordered set
for entry in document["entries"]:
chaps[entry["chapter"]] = None
if entry["time"] > time:
time = entry["time"]
return time, [elm for elm in chaps]
from datetime import datetime
import hashlib
def koreader_entry(entry, level=2):
time = datetime.fromtimestamp(entry.get("time", 0))
time_s = datetime.strftime(time, "[%Y-%m-%d %a %H:%M]")
time_s2 = datetime.strftime(time, "%Y-%m-%d %H:%M:%S") # used in digest
text = entry["text"].replace("\n", "ΒΆ ")
digest = hashlib.sha256(text.encode('utf-8')+time_s2.encode('utf-8')).hexdigest().upper()
page = entry.get("page", "")
data = entry.update(dict(
id=digest,
text=text,
time=time,
time_s=time_s
))
return ("""{level} {text}
:PROPERTIES:
:ID: {id}
:PAGE: {page}
:END:
{time}
""".format(
level="*" * level,
time=time_s,
text=text,
id=digest,
page=page
), entry)
import pathlib
def koreader_doc(document):
latest_time, chapters = collect_chapters(document)
level = 1
if len(chapters) > 1:
level = 2
entries = [ koreader_entry(entry,level) for entry in document["entries"] ]
curr_chapter = ""
filename = document["title"].replace(" ", "_").replace("\0", "") + ".org"
filepath = pathlib.Path(args.dir, pathlib.Path(filename)).expanduser()
docheader = """:PROPERTIES:
:ID: {id}
:END:
,#+TITLE: Notes from {title}
,#+AUTHOR: {author}
[[file:{sourcepath}][{sourcepath}]]
""".format(id="koreader-" + document["md5sum"],
sourcepath=document["file"],
title=document["title"],
author=document["author"],
)
out = """{docheader}
""".format(docheader=docheader)
for entry in entries:
if len(chapters) > 1 and curr_chapter != entry[1]["chapter"]:
curr_chapter = entry[1]["chapter"]
out += "* {curr}\n".format(curr=curr_chapter)
out += entry[0]
if args.force:
write_file(filepath, out)
elif not filepath.is_file():
write_file(filepath, out)
elif filepath.is_file() and filepath.stat().st_mtime < latest_time:
write_file(filepath, out)
else:
print("Skip {}".format(filepath))
def write_file(filepath, out):
print(filepath)
if args.dry_run:
print("="*39 + "8<" + "="*39)
print(out)
else:
with filepath.open('w') as f:
f.write(out)
def main():
import json
with open(args.filename, 'r') as f:
jason = json.load(f)
[ koreader_doc(doc) for doc in jason["documents"] ]
main()Importing KOreader notes with Local parsing of KOReader Notes through Fennel
(deprecated) Importing KOreader highlights in to org-roam with Python
to extract the clipping file to my local machine...
They used to be accessed over a usbnet SSH, but now Syncthing puts my koreader configuration in sync between tablet and desktop. on My NixOS this file will be generated by running koreader & and navigating to the export
#scp -o "pubkeyacceptedkeytypes +ssh-rsa" root@remarkable:/opt/koreader/clipboard/KOReaderClipping.json /home/rrix/org/remarkable_notes/koreader.json
cp ~/.config/koreader/clipboard/KOReaderClipping.json ~/org/remarkable_notes/koreader.json
ls -alh /home/rrix/org/remarkable_notes/koreader.json: -rw-r--r-- 1 rrix humans 8.2M Jan 11 14:34 /home/rrix/org/remarkable_notes/koreader.json
This file contains: a list of books with metadata keys, and numeric keys for the bookmarks. whee.
restructuring KOReaderClipping.json for consumption by the importer
This can be processed with something like jq probably, but I'll just use Python so that it can be easily integrated in to the automation. Invoking this function is straightforward and returns a dict object.
import json
def restructure_notes(fname):
raw = None # file data here
jason = [] # decoded json dicts here, we call this jason because it looks like json
with open(fname, 'r') as f:
for l in f.readlines():
jason.append(json.loads(l))
massaged = []
import re
for book in jason:
reshape = {
'author': book['author'],
'title': book['title'],
'path': book['file']
}
if 'w-id_' in reshape['path']: # special case wallabag highlights
m = re.search(r'\[w-id_([0-9]+)\]\s+(.*).epub$', reshape['path'])
wbid = m.group(1)
reshape['title'] = m.group(2)
reshape['author'] = 'Wallabag'
reshape['wallabag'] = wbid
# god this json sucks.
for k, v in book.items():
try:
index = int(k)
note = book[k][0]
chapter = note.get('chapter', "no_chap")
notes = reshape.get('notes', {})
notes[chapter] = notes.get(chapter, [])
notes[chapter].append(note)
reshape['notes'] = notes
except ValueError as e:
pass
massaged.append(reshape)
return massagedThe notes are shaped like:
#+begin_exmple [ { metadata_keys, notes: { 'chapter_1': [ {keys 'page', 'text', 'time', 'sort' } ], } } ] #+end_exmple
org-mode export of the KOReaderClipping file
and that can turn in to org-mode files pretty easily; the heading text is kind of wild because Org Babel will indent the docstrings if I run this code in normal literate mode, and it does misformatting of out-dents if I use session mode. Python support in org-babel isn't really great, but it's the language I reach for when I do data transformation like this for some reason. I don't even write very "Pythonic" code in general, I wish I had a different tool that I was as agile with. The logic below takes that JSON transformation from above, and writes the files out based on the title of the file; it probably makes more sense for this to export based on the filename on the tablet, it's in the metadata_keys... I probably could do a much better job templating the org-mode files, maybe using Memacs or orger ...
OUTPUT_DIRECTORY=~/org/remarkable_notes/
mkdir -p $OUTPUT_DIRECTORY
echo $OUTPUT_DIRECTORY: /home/rrix/org/remarkable_notes/
import re
from functools import reduce
import json
from datetime import datetime
import os.path as path
fname = '/home/rrix/org/remarkable_notes/koreader.json'
REGEXPS = [
(r'[^\w]', '_'),
(r'__*', '_'),
(r'^_', ''),
(r'_$', '_')
]
BOOK_HEADING = """#+TITLE: {title}
,#+ROAM_TAGS: reMarkable Archive
,#+FILETAGS: reMarkable Archive
[[id:912e0c27-c561-445e-813a-b0c6e416afa2][reMarkableTablet]] [[id:6975a285-eb1b-4531-bcfd-584de3ba8859][KOReader Notes]]
,*This is outside the Archive and my org-roam and should be ingested elsewhere.*
Author: {author}\n\n"""
CHAPTER_HEADING = "* {title}\n"
NOTE_HEADING = "** {time} Page {page}: {text}\n"
# import restructure_notes:
<<structure_notes>>
def to_org_time(ts):
dt = datetime.fromtimestamp(ts)
return dt.strftime("[%Y-%m-%d %a %H:%M]")
def to_slug(title):
title = title.lower()
for pair in REGEXPS:
title = re.sub(pair[0], pair[1], title)
return title
def title_to_path(book):
slug = to_slug(book['title'])
return path.join(OUTPUT_DIRECTORY, '{}.org'.format(slug))
def export_book(book, output_path):
wbid = None
with open(output_path, 'w') as f:
f.write(BOOK_HEADING.format(
title=book['title'],
author=book['author']
))
if book.get('wallabag'):
wbid = book.get('wallabag')
f.write("#+WALLABAG_ID: {}\n".format(wbid))
for chapter, notes in book['notes'].items():
f.write(CHAPTER_HEADING.format(
title=chapter
))
for note in notes:
f.write(NOTE_HEADING.format(
time=to_org_time(note['time']),
page=note['page'],
text=note['text']
))
if wbid:
return "[[{}][{}]] ([[wallabag:{}]])".format(output_path, book['title'], wbid)
return "[[{}][{}]]".format(output_path, book['title'])
return '\n'.join(sorted(set(
[
"- " + export_book(book, title_to_path(book))
for book in restructure_notes(fname)
]
)))NEXT work would be to add Wallabag API cross-work to get the URLs. The notes export feature will export notes for things which are even deleted from the library, it looks like as of which is a great thing.
These should generate idempotent IDs, this is really just an HPI or Memacs module...!