Skip to content

Commit

Permalink
Fixes silverbulletmd#529 by removing directives (silverbulletmd#613)
Browse files Browse the repository at this point in the history 
* Fixes silverbulletmd#529 by removing directives
* Load builtin tags on space reindex
  • Loading branch information
@zefhemel
zefhemel authored Jan 2, 2024
1 parent 5f4e584 commit 8a2e081
Show file tree
Hide file tree
Showing 46 changed files with 159 additions and 1,445 deletions.
4 changes: 2 additions & 2 deletions common/languages.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import {
xmlLanguage,
yamlLanguage,
} from "./deps.ts";
import { highlightingDirectiveParser } from "./markdown_parser/parser.ts";
import { highlightingQueryParser } from "./markdown_parser/parser.ts";

export const builtinLanguages: Record<string, Language> = {
"meta": StreamLanguage.define(yamlLanguage),
Expand Down Expand Up @@ -79,7 +79,7 @@ export const builtinLanguages: Record<string, Language> = {
"dart": StreamLanguage.define(dartLanguage),
"query": LRLanguage.define({
name: "query",
parser: highlightingDirectiveParser,
parser: highlightingQueryParser,
}),
};

Expand Down
5 changes: 0 additions & 5 deletions common/markdown_parser/customtags.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,6 @@ export const OrderedList = Tag.define();
export const Highlight = Tag.define();
export const HorizontalRuleTag = Tag.define();

export const DirectiveTag = Tag.define();
export const DirectiveStartTag = Tag.define();
export const DirectiveEndTag = Tag.define();
export const DirectiveProgramTag = Tag.define();

export const AttributeTag = Tag.define();
export const AttributeNameTag = Tag.define();
export const AttributeValueTag = Tag.define();
Expand Down
28 changes: 3 additions & 25 deletions common/markdown_parser/parser.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,28 +51,6 @@ Deno.test("Test parser", () => {
assertEquals(node, undefined);
});

const directiveSample = `
Before
<!-- #query page -->
Body line 1
Body line 2
<!-- /query -->
End
`;

const nestedDirectiveExample = `
Before
<!-- #query page -->
1
<!-- #eval 10 * 10 -->
100
<!-- /eval -->
3
<!-- /query -->
End
`;

const inlineAttributeSample = `
Hello there [a link](http://zef.plus)
[age: 100]
Expand Down Expand Up @@ -152,9 +130,9 @@ Deno.test("Test command link arguments", () => {
const commands = collectNodesOfType(tree, "CommandLink");
assertEquals(commands.length, 2);

const args1 = findNodeOfType(commands[0], "CommandLinkArgs")
const args1 = findNodeOfType(commands[0], "CommandLinkArgs");
assertEquals(args1!.children![0].text, '"with", "args"');

const args2 = findNodeOfType(commands[1], "CommandLinkArgs")
const args2 = findNodeOfType(commands[1], "CommandLinkArgs");
assertEquals(args2!.children![0].text, '"other", "args", 123');
});
});
129 changes: 15 additions & 114 deletions common/markdown_parser/parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,21 @@ export const Highlight: MarkdownConfig = {
],
};

import { parser as queryParser } from "./parse-query.js";

export const highlightingQueryParser = queryParser.configure({
props: [
styleTags({
"Name": t.variableName,
"String": t.string,
"Number": t.number,
"PageRef": ct.WikiLinkTag,
"where limit select render Order OrderKW and or as InKW each all":
t.keyword,
}),
],
});

export const attributeStartRegex = /^\[([\w\$]+)(::?\s*)/;

export const Attribute: MarkdownConfig = {
Expand Down Expand Up @@ -264,121 +279,8 @@ export const Comment: MarkdownConfig = {
],
};

// Directive parser

const directiveStart = /^\s*<!--\s*#([a-z]+)\s*(.*?)-->\s*/;
const directiveEnd = /^\s*<!--\s*\/(.*?)-->\s*/;

import { parser as directiveParser } from "./parse-query.js";
import { parser as expressionParser } from "./parse-expression.js";
import { Table } from "./table_parser.ts";
import { foldNodeProp } from "@codemirror/language";
import { lezerToParseTree } from "./parse_tree.ts";

export const highlightingDirectiveParser = directiveParser.configure({
props: [
styleTags({
"Name": t.variableName,
"String": t.string,
"Number": t.number,
"PageRef": ct.WikiLinkTag,
"where limit select render Order OrderKW and or as InKW each all":
t.keyword,
}),
],
});

export const Directive: MarkdownConfig = {
defineNodes: [
{ name: "Directive", block: true, style: ct.DirectiveTag },
{ name: "DirectiveStart", style: ct.DirectiveStartTag, block: true },
{ name: "DirectiveEnd", style: ct.DirectiveEndTag },
{ name: "DirectiveBody", block: true },
],
parseBlock: [{
name: "Directive",
parse: (cx, line: Line) => {
const match = directiveStart.exec(line.text);
if (!match) {
return false;
}

// console.log("Parsing directive", line.text);

const frontStart = cx.parsedPos;
const [fullMatch, directive, arg] = match;
const elts = [];
if (directive === "query") {
const queryParseTree = highlightingDirectiveParser.parse(arg);
elts.push(cx.elt(
"DirectiveStart",
cx.parsedPos,
cx.parsedPos + line.text.length + 1,
[cx.elt(queryParseTree, frontStart + fullMatch.indexOf(arg))],
));
} else if (directive === "eval") {
const expressionParseTree = expressionParser.parse(arg);
elts.push(cx.elt(
"DirectiveStart",
cx.parsedPos,
cx.parsedPos + line.text.length + 1,
[cx.elt(expressionParseTree, frontStart + fullMatch.indexOf(arg))],
));
} else {
elts.push(cx.elt(
"DirectiveStart",
cx.parsedPos,
cx.parsedPos + line.text.length + 1,
));
}

// console.log("Query parse tree", queryParseTree.topNode);

cx.nextLine();
const startPos = cx.parsedPos;
let endPos = startPos;
let text = "";
let lastPos = cx.parsedPos;
let nesting = 0;
while (true) {
if (directiveEnd.exec(line.text) && nesting === 0) {
break;
}
text += line.text + "\n";
endPos += line.text.length + 1;
if (directiveStart.exec(line.text)) {
nesting++;
}
if (directiveEnd.exec(line.text)) {
nesting--;
}
cx.nextLine();
if (cx.parsedPos === lastPos) {
// End of file, no progress made, there may be a better way to do this but :shrug:
return false;
}
lastPos = cx.parsedPos;
}
const directiveBodyTree = cx.parser.parse(text);

elts.push(
cx.elt("DirectiveBody", startPos, endPos, [
cx.elt(directiveBodyTree, startPos),
]),
);
endPos = cx.parsedPos + line.text.length;
elts.push(cx.elt(
"DirectiveEnd",
cx.parsedPos,
cx.parsedPos + line.text.length,
));
cx.nextLine();
cx.addElement(cx.elt("Directive", frontStart, endPos, elts));
return true;
},
before: "HTMLBlock",
}],
};

// FrontMatter parser

Expand Down Expand Up @@ -450,7 +352,6 @@ export default function buildMarkdown(mdExtensions: MDExt[]): Language {
CommandLink,
Attribute,
FrontMatter,
Directive,
TaskList,
Comment,
Highlight,
Expand Down
36 changes: 2 additions & 34 deletions common/spaces/sync.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { removeDirectiveBody, SpaceSync, SyncStatusItem } from "./sync.ts";
import { SpaceSync, SyncStatusItem } from "./sync.ts";
import { DiskSpacePrimitives } from "./disk_space_primitives.ts";
import { assertEquals } from "../../test_deps.ts";

Expand Down Expand Up @@ -117,23 +117,11 @@ Deno.test("Test store", async () => {
await primary.writeFile("index", stringToBytes("Hello 1"));
await secondary.writeFile("index", stringToBytes("Hello 1"));

// And two more files with different bodies, but only within a query directive — shouldn't conflict
await primary.writeFile(
"index.md",
stringToBytes(
"Hello\n<!-- #query page -->\nHello 1\n<!-- /query -->",
),
);
await secondary.writeFile(
"index.md",
stringToBytes("Hello\n<!-- #query page -->\nHello 2\n<!-- /query -->"),
);

await doSync();
await doSync();

// test + index + index.md + previous index.conflicting copy but nothing more
assertEquals((await primary.fetchFileList()).length, 4);
assertEquals((await primary.fetchFileList()).length, 3);

console.log("Bringing a third device in the mix");

Expand Down Expand Up @@ -191,26 +179,6 @@ function sleep(ms = 10): Promise<void> {
});
}

Deno.test("Remove directive bodies", () => {
assertEquals(
removeDirectiveBody(`<!-- #query page -->
This is a body
bla bla bla
<!-- /query -->
Hello
<!-- #include [[test]] -->
This is a body
<!-- /include -->
`),
`<!-- #query page -->
<!-- /query -->
Hello
<!-- #include [[test]] -->
<!-- /include -->
`,
);
});

function stringToBytes(s: string): Uint8Array {
return new TextEncoder().encode(s);
}
80 changes: 19 additions & 61 deletions common/spaces/sync.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
import { renderToText, replaceNodesMatching } from "../../plug-api/lib/tree.ts";
import buildMarkdown from "../markdown_parser/parser.ts";
import { parse } from "../markdown_parser/parse_tree.ts";
import { SpacePrimitives } from "./space_primitives.ts";
import { EventEmitter } from "../../plugos/event.ts";
import { FileMeta } from "$sb/types.ts";
Expand Down Expand Up @@ -305,56 +302,32 @@ export class SpaceSync extends EventEmitter<SyncEvents> {
const pageData1 = await primary.readFile(name);
const pageData2 = await secondary.readFile(name);

if (name.endsWith(".md")) {
console.log(
"[sync]",
"File is markdown, using smart conflict resolution",
);
// Let's use a smartert check for markdown files, ignoring directive bodies
const pageText1 = removeDirectiveBody(
new TextDecoder().decode(pageData1.data),
);
const pageText2 = removeDirectiveBody(
new TextDecoder().decode(pageData2.data),
);
if (pageText1 === pageText2) {
console.log(
"[sync]",
"Files are the same (eliminating the directive bodies), no conflict",
);
let byteWiseMatch = true;
const arrayBuffer1 = pageData1.data;
const arrayBuffer2 = pageData2.data;
if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) {
byteWiseMatch = false;
}
if (byteWiseMatch) {
// Byte-wise comparison
for (let i = 0; i < arrayBuffer1.byteLength; i++) {
if (arrayBuffer1[i] !== arrayBuffer2[i]) {
byteWiseMatch = false;
break;
}
}
// Byte wise they're still the same, so no confict
if (byteWiseMatch) {
console.log("[sync]", "Files are the same, no conflict");

snapshot.set(name, [
pageData1.meta.lastModified,
pageData2.meta.lastModified,
]);
return 0;
}
} else {
let byteWiseMatch = true;
const arrayBuffer1 = pageData1.data;
const arrayBuffer2 = pageData2.data;
if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) {
byteWiseMatch = false;
}
if (byteWiseMatch) {
// Byte-wise comparison
for (let i = 0; i < arrayBuffer1.byteLength; i++) {
if (arrayBuffer1[i] !== arrayBuffer2[i]) {
byteWiseMatch = false;
break;
}
}
// Byte wise they're still the same, so no confict
if (byteWiseMatch) {
console.log("[sync]", "Files are the same, no conflict");

snapshot.set(name, [
pageData1.meta.lastModified,
pageData2.meta.lastModified,
]);
return 0;
}
}
}

let operations = 0;
const revisionFileName = filePieces.length === 1
? `${name}.conflicted.${pageData2.meta.lastModified}`
Expand Down Expand Up @@ -403,18 +376,3 @@ export class SpaceSync extends EventEmitter<SyncEvents> {
}
}
}

const markdownLanguage = buildMarkdown([]);

export function removeDirectiveBody(text: string): string {
// Parse
const tree = parse(markdownLanguage, text);
// Remove bodies
replaceNodesMatching(tree, (node) => {
if (node.type === "DirectiveBody") {
return null;
}
});
// Turn back into text
return renderToText(tree);
}
Loading

0 comments on commit 8a2e081

Please sign in to comment.