Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

BREAKING CHANGE: clean up rules so they can be typed #3087

Merged
merged 10 commits into from
Nov 29, 2023
Merged

BREAKING CHANGE: clean up rules so they can be typed #3087

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ba5dfa7
type rules
UziTech Nov 13, 2023
542286d
fix tokenizer types
UziTech Nov 13, 2023
9fcb4d9
remove only
UziTech Nov 13, 2023
32c4661
set tokens type
UziTech Nov 13, 2023
23578e2
clean up rules
UziTech Nov 13, 2023
c7d5ac2
fix rules type test
UziTech Nov 13, 2023
83ba7ac
clean up code
UziTech Nov 13, 2023
d95fca5
clean up options types
UziTech Nov 13, 2023
c03be9b
clean up emstrong
UziTech Nov 14, 2023
7385b3a
clean up rules
UziTech Nov 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 23 additions & 13 deletions src/Instance.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,11 +142,14 @@ export class Marked {
if (pack.renderer) {
const renderer = this.defaults.renderer || new _Renderer(this.defaults);
for (const prop in pack.renderer) {
const rendererFunc = pack.renderer[prop as keyof MarkedExtension['renderer']] as GenericRendererFunction;
const rendererKey = prop as keyof _Renderer;
const prevRenderer = renderer[rendererKey] as GenericRendererFunction;
if (!(prop in renderer) || prop === 'options') {
throw new Error(`renderer '${prop}' does not exist`);
}
const rendererProp = prop as Exclude<keyof _Renderer, 'options'>;
const rendererFunc = pack.renderer[rendererProp] as GenericRendererFunction;
const prevRenderer = renderer[rendererProp] as GenericRendererFunction;
// Replace renderer with func to run extension, but fall back if false
renderer[rendererKey] = (...args: unknown[]) => {
renderer[rendererProp] = (...args: unknown[]) => {
let ret = rendererFunc.apply(renderer, args);
if (ret === false) {
ret = prevRenderer.apply(renderer, args);
Expand All @@ -159,11 +162,15 @@ export class Marked {
if (pack.tokenizer) {
const tokenizer = this.defaults.tokenizer || new _Tokenizer(this.defaults);
for (const prop in pack.tokenizer) {
const tokenizerFunc = pack.tokenizer[prop as keyof MarkedExtension['tokenizer']] as UnknownFunction;
const tokenizerKey = prop as keyof _Tokenizer;
const prevTokenizer = tokenizer[tokenizerKey] as UnknownFunction;
if (!(prop in tokenizer) || ['options', 'rules', 'lexer'].includes(prop)) {
throw new Error(`tokenizer '${prop}' does not exist`);
}
const tokenizerProp = prop as Exclude<keyof _Tokenizer, 'options' | 'rules' | 'lexer'>;
const tokenizerFunc = pack.tokenizer[tokenizerProp] as UnknownFunction;
const prevTokenizer = tokenizer[tokenizerProp] as UnknownFunction;
// Replace tokenizer with func to run extension, but fall back if false
tokenizer[tokenizerKey] = (...args: unknown[]) => {
// @ts-expect-error cannot type tokenizer function dynamically
tokenizer[tokenizerProp] = (...args: unknown[]) => {
let ret = tokenizerFunc.apply(tokenizer, args);
if (ret === false) {
ret = prevTokenizer.apply(tokenizer, args);
Expand All @@ -178,11 +185,14 @@ export class Marked {
if (pack.hooks) {
const hooks = this.defaults.hooks || new _Hooks();
for (const prop in pack.hooks) {
const hooksFunc = pack.hooks[prop as keyof MarkedExtension['hooks']] as UnknownFunction;
const hooksKey = prop as keyof _Hooks;
const prevHook = hooks[hooksKey] as UnknownFunction;
if (!(prop in hooks) || prop === 'options') {
throw new Error(`hook '${prop}' does not exist`);
}
const hooksProp = prop as Exclude<keyof _Hooks, 'options'>;
const hooksFunc = pack.hooks[hooksProp] as UnknownFunction;
const prevHook = hooks[hooksProp] as UnknownFunction;
if (_Hooks.passThroughHooks.has(prop)) {
hooks[hooksKey as 'preprocess' | 'postprocess'] = (arg: string | undefined) => {
hooks[hooksProp] = (arg: string | undefined) => {
if (this.defaults.async) {
return Promise.resolve(hooksFunc.call(hooks, arg)).then(ret => {
return prevHook.call(hooks, ret) as string;
Expand All @@ -193,7 +203,7 @@ export class Marked {
return prevHook.call(hooks, ret) as string;
};
} else {
hooks[hooksKey] = (...args: unknown[]) => {
hooks[hooksProp] = (...args: unknown[]) => {
let ret = hooksFunc.apply(hooks, args);
if (ret === false) {
ret = prevHook.apply(hooks, args);
Expand Down
6 changes: 2 additions & 4 deletions src/Lexer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { _defaults } from './defaults.ts';
import { block, inline } from './rules.ts';
import type { Token, TokensList, Tokens } from './Tokens.ts';
import type { MarkedOptions, TokenizerExtension } from './MarkedOptions.ts';
import type { Rules } from './rules.ts';

/**
* Block Lexer
Expand All @@ -22,8 +21,7 @@ export class _Lexer {

constructor(options?: MarkedOptions) {
// TokenList cannot be created in one go
// @ts-expect-error
this.tokens = [];
this.tokens = [] as unknown as TokensList;
this.tokens.links = Object.create(null);
this.options = options || _defaults;
this.options.tokenizer = this.options.tokenizer || new _Tokenizer();
Expand Down Expand Up @@ -59,7 +57,7 @@ export class _Lexer {
/**
* Expose Rules
*/
static get rules(): Rules {
static get rules() {
return {
block,
inline
Expand Down
134 changes: 64 additions & 70 deletions src/Tokenizer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
escape,
findClosingBracket
} from './helpers.ts';
import type { Rules } from './rules.ts';
import type { _Lexer } from './Lexer.ts';
import type { Links, Tokens } from './Tokens.ts';
import type { MarkedOptions } from './MarkedOptions.ts';
Expand Down Expand Up @@ -69,9 +70,8 @@ function indentCodeCompensation(raw: string, text: string) {
*/
export class _Tokenizer {
options: MarkedOptions;
// TODO: Fix this rules type
rules: any;
lexer!: _Lexer;
rules!: Rules; // set by the lexer
lexer!: _Lexer; // set by the lexer

constructor(options?: MarkedOptions) {
this.options = options || _defaults;
Expand Down Expand Up @@ -111,7 +111,7 @@ export class _Tokenizer {
return {
type: 'code',
raw,
lang: cap[2] ? cap[2].trim().replace(this.rules.inline._escapes, '$1') : cap[2],
lang: cap[2] ? cap[2].trim().replace(this.rules.inline.anyPunctuation, '$1') : cap[2],
text
};
}
Expand Down Expand Up @@ -182,7 +182,7 @@ export class _Tokenizer {
ordered: isordered,
start: isordered ? +bull.slice(0, -1) : '',
loose: false,
items: [] as Tokens.ListItem[]
items: []
};

bull = isordered ? `\\d{1,9}\\${bull.slice(-1)}` : `\\${bull}`;
Expand All @@ -207,10 +207,10 @@ export class _Tokenizer {
break;
}

raw = cap[0] as string;
raw = cap[0];
src = src.substring(raw.length);

let line = cap[2].split('\n', 1)[0].replace(/^\t+/, (t: string) => ' '.repeat(3 * t.length)) as string;
let line = cap[2].split('\n', 1)[0].replace(/^\t+/, (t: string) => ' '.repeat(3 * t.length));
let nextLine = src.split('\n', 1)[0];

let indent = 0;
Expand Down Expand Up @@ -338,7 +338,7 @@ export class _Tokenizer {

// Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic
list.items[list.items.length - 1].raw = raw.trimEnd();
(list.items[list.items.length - 1] as Tokens.ListItem).text = itemContents.trimEnd();
(list.items[list.items.length - 1]).text = itemContents.trimEnd();
list.raw = list.raw.trimEnd();

// Item child tokens handled here at end because we needed to have the final item to trim it first
Expand Down Expand Up @@ -384,8 +384,8 @@ export class _Tokenizer {
const cap = this.rules.block.def.exec(src);
if (cap) {
const tag = cap[1].toLowerCase().replace(/\s+/g, ' ');
const href = cap[2] ? cap[2].replace(/^<(.*)>$/, '$1').replace(this.rules.inline._escapes, '$1') : '';
const title = cap[3] ? cap[3].substring(1, cap[3].length - 1).replace(this.rules.inline._escapes, '$1') : cap[3];
const href = cap[2] ? cap[2].replace(/^<(.*)>$/, '$1').replace(this.rules.inline.anyPunctuation, '$1') : '';
const title = cap[3] ? cap[3].substring(1, cap[3].length - 1).replace(this.rules.inline.anyPunctuation, '$1') : cap[3];
return {
type: 'def',
tag,
Expand All @@ -398,67 +398,61 @@ export class _Tokenizer {

table(src: string): Tokens.Table | undefined {
const cap = this.rules.block.table.exec(src);
if (cap) {
if (!/[:|]/.test(cap[2])) {
// delimiter row must have a pipe (|) or colon (:) otherwise it is a setext heading
return;
}
if (!cap) {
return;
}

const item: Tokens.Table = {
type: 'table',
raw: cap[0],
header: splitCells(cap[1]).map(c => {
return { text: c, tokens: [] };
}),
align: cap[2].replace(/^\||\| *$/g, '').split('|'),
rows: cap[3] && cap[3].trim() ? cap[3].replace(/\n[ \t]*$/, '').split('\n') : []
};
if (!/[:|]/.test(cap[2])) {
// delimiter row must have a pipe (|) or colon (:) otherwise it is a setext heading
return;
}

if (item.header.length === item.align.length) {
let l = item.align.length;
let i, j, k, row;
for (i = 0; i < l; i++) {
const align = item.align[i];
if (align) {
if (/^ *-+: *$/.test(align)) {
item.align[i] = 'right';
} else if (/^ *:-+: *$/.test(align)) {
item.align[i] = 'center';
} else if (/^ *:-+ *$/.test(align)) {
item.align[i] = 'left';
} else {
item.align[i] = null;
}
}
}
const headers = splitCells(cap[1]);
const aligns = cap[2].replace(/^\||\| *$/g, '').split('|');
const rows = cap[3] && cap[3].trim() ? cap[3].replace(/\n[ \t]*$/, '').split('\n') : [];

l = item.rows.length;
for (i = 0; i < l; i++) {
item.rows[i] = splitCells(item.rows[i] as unknown as string, item.header.length).map(c => {
return { text: c, tokens: [] };
});
}
const item: Tokens.Table = {
type: 'table',
raw: cap[0],
header: [],
align: [],
rows: []
};

// parse child tokens inside headers and cells
if (headers.length !== aligns.length) {
// header and align columns must be equal, rows can be different.
return;
}

// header child tokens
l = item.header.length;
for (j = 0; j < l; j++) {
item.header[j].tokens = this.lexer.inline(item.header[j].text);
}
for (const align of aligns) {
if (/^ *-+: *$/.test(align)) {
item.align.push('right');
} else if (/^ *:-+: *$/.test(align)) {
item.align.push('center');
} else if (/^ *:-+ *$/.test(align)) {
item.align.push('left');
} else {
item.align.push(null);
}
}

// cell child tokens
l = item.rows.length;
for (j = 0; j < l; j++) {
row = item.rows[j];
for (k = 0; k < row.length; k++) {
row[k].tokens = this.lexer.inline(row[k].text);
}
}
for (const header of headers) {
item.header.push({
text: header,
tokens: this.lexer.inline(header)
});
}

return item;
}
for (const row of rows) {
item.rows.push(splitCells(row, item.header.length).map(cell => {
return {
text: cell,
tokens: this.lexer.inline(cell)
};
}));
}

return item;
}

lheading(src: string): Tokens.Heading | undefined {
Expand Down Expand Up @@ -587,8 +581,8 @@ export class _Tokenizer {
}
}
return outputLink(cap, {
href: href ? href.replace(this.rules.inline._escapes, '$1') : href,
title: title ? title.replace(this.rules.inline._escapes, '$1') : title
href: href ? href.replace(this.rules.inline.anyPunctuation, '$1') : href,
title: title ? title.replace(this.rules.inline.anyPunctuation, '$1') : title
}, cap[0], this.lexer);
}
}
Expand All @@ -597,8 +591,8 @@ export class _Tokenizer {
let cap;
if ((cap = this.rules.inline.reflink.exec(src))
|| (cap = this.rules.inline.nolink.exec(src))) {
let link = (cap[2] || cap[1]).replace(/\s+/g, ' ');
link = links[link.toLowerCase()];
const linkString = (cap[2] || cap[1]).replace(/\s+/g, ' ');
const link = links[linkString.toLowerCase()];
if (!link) {
const text = cap[0].charAt(0);
return {
Expand All @@ -612,7 +606,7 @@ export class _Tokenizer {
}

emStrong(src: string, maskedSrc: string, prevChar = ''): Tokens.Em | Tokens.Strong | undefined {
let match = this.rules.inline.emStrong.lDelim.exec(src);
let match = this.rules.inline.emStrongLDelim.exec(src);
if (!match) return;

// _ can't be between two alphanumerics. \p{L}\p{N} includes non-english alphabet/numbers as well
Expand All @@ -625,7 +619,7 @@ export class _Tokenizer {
const lLength = [...match[0]].length - 1;
let rDelim, rLength, delimTotal = lLength, midDelimTotal = 0;

const endReg = match[0][0] === '*' ? this.rules.inline.emStrong.rDelimAst : this.rules.inline.emStrong.rDelimUnd;
const endReg = match[0][0] === '*' ? this.rules.inline.emStrongRDelimAst : this.rules.inline.emStrongRDelimUnd;
endReg.lastIndex = 0;

// Clip maskedSrc to same section of string as src (move to lexer?)
Expand Down Expand Up @@ -761,7 +755,7 @@ export class _Tokenizer {
let prevCapZero;
do {
prevCapZero = cap[0];
cap[0] = this.rules.inline._backpedal.exec(cap[0])[0];
cap[0] = this.rules.inline._backpedal.exec(cap[0])?.[0] ?? '';
} while (prevCapZero !== cap[0]);
text = escape(cap[0]);
if (cap[1] === 'www.') {
Expand Down
16 changes: 7 additions & 9 deletions src/helpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import type { Rule } from './rules.ts';

/**
* Helpers
*/
Expand Down Expand Up @@ -48,18 +46,18 @@ export function unescape(html: string) {

const caret = /(^|[^\[])\^/g;

export function edit(regex: Rule, opt?: string) {
regex = typeof regex === 'string' ? regex : regex.source;
export function edit(regex: string | RegExp, opt?: string) {
let source = typeof regex === 'string' ? regex : regex.source;
opt = opt || '';
const obj = {
replace: (name: string | RegExp, val: string | RegExp) => {
val = typeof val === 'object' && 'source' in val ? val.source : val;
val = val.replace(caret, '$1');
regex = (regex as string).replace(name, val);
let valSource = typeof val === 'string' ? val : val.source;
valSource = valSource.replace(caret, '$1');
source = source.replace(name, valSource);
return obj;
},
getRegex: () => {
return new RegExp(regex, opt);
return new RegExp(source, opt);
}
};
return obj;
Expand All @@ -74,7 +72,7 @@ export function cleanUrl(href: string) {
return href;
}

export const noopTest = { exec: () => null };
export const noopTest = { exec: () => null } as unknown as RegExp;

export function splitCells(tableRow: string, count?: number) {
// ensure that every cell-delimiting pipe has a space
Expand Down
Loading