admin/test
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Erster Docker-Stand

Browse Source
This commit is contained in:
Ali
2026-02-20 16:06:40 +09:00
commit f31e2e8ed3
8818 changed files with 1605323 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

951
_node_modules/chevrotain/lib/src/scan/lexer.js generated Normal file
View File

@@ -0,0 +1,951 @@
"use strict";
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
if (typeof b !== "function" && b !== null)
throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.charCodeToOptimizedIndex = exports.minOptimizationVal = exports.buildLineBreakIssueMessage = exports.LineTerminatorOptimizedTester = exports.isShortPattern = exports.isCustomPattern = exports.cloneEmptyGroups = exports.performWarningRuntimeChecks = exports.performRuntimeChecks = exports.addStickyFlag = exports.addStartOfInput = exports.findUnreachablePatterns = exports.findModesThatDoNotExist = exports.findInvalidGroupType = exports.findDuplicatePatterns = exports.findUnsupportedFlags = exports.findStartOfInputAnchor = exports.findEmptyMatchRegExps = exports.findEndOfInputAnchor = exports.findInvalidPatterns = exports.findMissingPatterns = exports.validatePatterns = exports.analyzeTokenTypes = exports.enableSticky = exports.disableSticky = exports.SUPPORT_STICKY = exports.MODES = exports.DEFAULT_MODE = void 0;
var regexp_to_ast_1 = require("regexp-to-ast");
var lexer_public_1 = require("./lexer_public");
var first_1 = __importDefault(require("lodash/first"));
var isEmpty_1 = __importDefault(require("lodash/isEmpty"));
var compact_1 = __importDefault(require("lodash/compact"));
var isArray_1 = __importDefault(require("lodash/isArray"));
var values_1 = __importDefault(require("lodash/values"));
var flatten_1 = __importDefault(require("lodash/flatten"));
var reject_1 = __importDefault(require("lodash/reject"));
var difference_1 = __importDefault(require("lodash/difference"));
var indexOf_1 = __importDefault(require("lodash/indexOf"));
var map_1 = __importDefault(require("lodash/map"));
var forEach_1 = __importDefault(require("lodash/forEach"));
var isString_1 = __importDefault(require("lodash/isString"));
var isFunction_1 = __importDefault(require("lodash/isFunction"));
var isUndefined_1 = __importDefault(require("lodash/isUndefined"));
var find_1 = __importDefault(require("lodash/find"));
var has_1 = __importDefault(require("lodash/has"));
var keys_1 = __importDefault(require("lodash/keys"));
var isRegExp_1 = __importDefault(require("lodash/isRegExp"));
var filter_1 = __importDefault(require("lodash/filter"));
var defaults_1 = __importDefault(require("lodash/defaults"));
var reduce_1 = __importDefault(require("lodash/reduce"));
var includes_1 = __importDefault(require("lodash/includes"));
var utils_1 = require("@chevrotain/utils");
var reg_exp_1 = require("./reg_exp");
var reg_exp_parser_1 = require("./reg_exp_parser");
var PATTERN = "PATTERN";
exports.DEFAULT_MODE = "defaultMode";
exports.MODES = "modes";
exports.SUPPORT_STICKY = typeof new RegExp("(?:)").sticky === "boolean";
function disableSticky() {
exports.SUPPORT_STICKY = false;
}
exports.disableSticky = disableSticky;
function enableSticky() {
exports.SUPPORT_STICKY = true;
}
exports.enableSticky = enableSticky;
function analyzeTokenTypes(tokenTypes, options) {
options = (0, defaults_1.default)(options, {
useSticky: exports.SUPPORT_STICKY,
debug: false,
safeMode: false,
positionTracking: "full",
lineTerminatorCharacters: ["\r", "\n"],
tracer: function (msg, action) { return action(); }
});
var tracer = options.tracer;
tracer("initCharCodeToOptimizedIndexMap", function () {
initCharCodeToOptimizedIndexMap();
});
var onlyRelevantTypes;
tracer("Reject Lexer.NA", function () {
onlyRelevantTypes = (0, reject_1.default)(tokenTypes, function (currType) {
return currType[PATTERN] === lexer_public_1.Lexer.NA;
});
});
var hasCustom = false;
var allTransformedPatterns;
tracer("Transform Patterns", function () {
hasCustom = false;
allTransformedPatterns = (0, map_1.default)(onlyRelevantTypes, function (currType) {
var currPattern = currType[PATTERN];
/* istanbul ignore else */
if ((0, isRegExp_1.default)(currPattern)) {
var regExpSource = currPattern.source;
if (regExpSource.length === 1 &&
// only these regExp meta characters which can appear in a length one regExp
regExpSource !== "^" &&
regExpSource !== "$" &&
regExpSource !== "." &&
!currPattern.ignoreCase) {
return regExpSource;
}
else if (regExpSource.length === 2 &&
regExpSource[0] === "\\" &&
// not a meta character
!(0, includes_1.default)([
"d",
"D",
"s",
"S",
"t",
"r",
"n",
"t",
"0",
"c",
"b",
"B",
"f",
"v",
"w",
"W"
], regExpSource[1])) {
// escaped meta Characters: /\+/ /\[/
// or redundant escaping: /\a/
// without the escaping "\"
return regExpSource[1];
}
else {
return options.useSticky
? addStickyFlag(currPattern)
: addStartOfInput(currPattern);
}
}
else if ((0, isFunction_1.default)(currPattern)) {
hasCustom = true;
// CustomPatternMatcherFunc - custom patterns do not require any transformations, only wrapping in a RegExp Like object
return { exec: currPattern };
}
else if (typeof currPattern === "object") {
hasCustom = true;
// ICustomPattern
return currPattern;
}
else if (typeof currPattern === "string") {
if (currPattern.length === 1) {
return currPattern;
}
else {
var escapedRegExpString = currPattern.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
var wrappedRegExp = new RegExp(escapedRegExpString);
return options.useSticky
? addStickyFlag(wrappedRegExp)
: addStartOfInput(wrappedRegExp);
}
}
else {
throw Error("non exhaustive match");
}
});
});
var patternIdxToType;
var patternIdxToGroup;
var patternIdxToLongerAltIdxArr;
var patternIdxToPushMode;
var patternIdxToPopMode;
tracer("misc mapping", function () {
patternIdxToType = (0, map_1.default)(onlyRelevantTypes, function (currType) { return currType.tokenTypeIdx; });
patternIdxToGroup = (0, map_1.default)(onlyRelevantTypes, function (clazz) {
var groupName = clazz.GROUP;
/* istanbul ignore next */
if (groupName === lexer_public_1.Lexer.SKIPPED) {
return undefined;
}
else if ((0, isString_1.default)(groupName)) {
return groupName;
}
else if ((0, isUndefined_1.default)(groupName)) {
return false;
}
else {
throw Error("non exhaustive match");
}
});
patternIdxToLongerAltIdxArr = (0, map_1.default)(onlyRelevantTypes, function (clazz) {
var longerAltType = clazz.LONGER_ALT;
if (longerAltType) {
var longerAltIdxArr = (0, isArray_1.default)(longerAltType)
? (0, map_1.default)(longerAltType, function (type) { return (0, indexOf_1.default)(onlyRelevantTypes, type); })
: [(0, indexOf_1.default)(onlyRelevantTypes, longerAltType)];
return longerAltIdxArr;
}
});
patternIdxToPushMode = (0, map_1.default)(onlyRelevantTypes, function (clazz) { return clazz.PUSH_MODE; });
patternIdxToPopMode = (0, map_1.default)(onlyRelevantTypes, function (clazz) {
return (0, has_1.default)(clazz, "POP_MODE");
});
});
var patternIdxToCanLineTerminator;
tracer("Line Terminator Handling", function () {
var lineTerminatorCharCodes = getCharCodes(options.lineTerminatorCharacters);
patternIdxToCanLineTerminator = (0, map_1.default)(onlyRelevantTypes, function (tokType) { return false; });
if (options.positionTracking !== "onlyOffset") {
patternIdxToCanLineTerminator = (0, map_1.default)(onlyRelevantTypes, function (tokType) {
if ((0, has_1.default)(tokType, "LINE_BREAKS")) {
return !!tokType.LINE_BREAKS;
}
else {
return (checkLineBreaksIssues(tokType, lineTerminatorCharCodes) === false &&
(0, reg_exp_1.canMatchCharCode)(lineTerminatorCharCodes, tokType.PATTERN));
}
});
}
});
var patternIdxToIsCustom;
var patternIdxToShort;
var emptyGroups;
var patternIdxToConfig;
tracer("Misc Mapping #2", function () {
patternIdxToIsCustom = (0, map_1.default)(onlyRelevantTypes, isCustomPattern);
patternIdxToShort = (0, map_1.default)(allTransformedPatterns, isShortPattern);
emptyGroups = (0, reduce_1.default)(onlyRelevantTypes, function (acc, clazz) {
var groupName = clazz.GROUP;
if ((0, isString_1.default)(groupName) && !(groupName === lexer_public_1.Lexer.SKIPPED)) {
acc[groupName] = [];
}
return acc;
}, {});
patternIdxToConfig = (0, map_1.default)(allTransformedPatterns, function (x, idx) {
return {
pattern: allTransformedPatterns[idx],
longerAlt: patternIdxToLongerAltIdxArr[idx],
canLineTerminator: patternIdxToCanLineTerminator[idx],
isCustom: patternIdxToIsCustom[idx],
short: patternIdxToShort[idx],
group: patternIdxToGroup[idx],
push: patternIdxToPushMode[idx],
pop: patternIdxToPopMode[idx],
tokenTypeIdx: patternIdxToType[idx],
tokenType: onlyRelevantTypes[idx]
};
});
});
var canBeOptimized = true;
var charCodeToPatternIdxToConfig = [];
if (!options.safeMode) {
tracer("First Char Optimization", function () {
charCodeToPatternIdxToConfig = (0, reduce_1.default)(onlyRelevantTypes, function (result, currTokType, idx) {
if (typeof currTokType.PATTERN === "string") {
var charCode = currTokType.PATTERN.charCodeAt(0);
var optimizedIdx = charCodeToOptimizedIndex(charCode);
addToMapOfArrays(result, optimizedIdx, patternIdxToConfig[idx]);
}
else if ((0, isArray_1.default)(currTokType.START_CHARS_HINT)) {
var lastOptimizedIdx_1;
(0, forEach_1.default)(currTokType.START_CHARS_HINT, function (charOrInt) {
var charCode = typeof charOrInt === "string"
? charOrInt.charCodeAt(0)
: charOrInt;
var currOptimizedIdx = charCodeToOptimizedIndex(charCode);
// Avoid adding the config multiple times
/* istanbul ignore else */
// - Difficult to check this scenario effects as it is only a performance
// optimization that does not change correctness
if (lastOptimizedIdx_1 !== currOptimizedIdx) {
lastOptimizedIdx_1 = currOptimizedIdx;
addToMapOfArrays(result, currOptimizedIdx, patternIdxToConfig[idx]);
}
});
}
else if ((0, isRegExp_1.default)(currTokType.PATTERN)) {
if (currTokType.PATTERN.unicode) {
canBeOptimized = false;
if (options.ensureOptimizations) {
(0, utils_1.PRINT_ERROR)("".concat(reg_exp_1.failedOptimizationPrefixMsg) +
"\tUnable to analyze < ".concat(currTokType.PATTERN.toString(), " > pattern.\n") +
"\tThe regexp unicode flag is not currently supported by the regexp-to-ast library.\n" +
"\tThis will disable the lexer's first char optimizations.\n" +
"\tFor details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#UNICODE_OPTIMIZE");
}
}
else {
var optimizedCodes = (0, reg_exp_1.getOptimizedStartCodesIndices)(currTokType.PATTERN, options.ensureOptimizations);
/* istanbul ignore if */
// start code will only be empty given an empty regExp or failure of regexp-to-ast library
// the first should be a different validation and the second cannot be tested.
if ((0, isEmpty_1.default)(optimizedCodes)) {
// we cannot understand what codes may start possible matches
// The optimization correctness requires knowing start codes for ALL patterns.
// Not actually sure this is an error, no debug message
canBeOptimized = false;
}
(0, forEach_1.default)(optimizedCodes, function (code) {
addToMapOfArrays(result, code, patternIdxToConfig[idx]);
});
}
}
else {
if (options.ensureOptimizations) {
(0, utils_1.PRINT_ERROR)("".concat(reg_exp_1.failedOptimizationPrefixMsg) +
"\tTokenType: <".concat(currTokType.name, "> is using a custom token pattern without providing <start_chars_hint> parameter.\n") +
"\tThis will disable the lexer's first char optimizations.\n" +
"\tFor details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#CUSTOM_OPTIMIZE");
}
canBeOptimized = false;
}
return result;
}, []);
});
}
return {
emptyGroups: emptyGroups,
patternIdxToConfig: patternIdxToConfig,
charCodeToPatternIdxToConfig: charCodeToPatternIdxToConfig,
hasCustom: hasCustom,
canBeOptimized: canBeOptimized
};
}
exports.analyzeTokenTypes = analyzeTokenTypes;
function validatePatterns(tokenTypes, validModesNames) {
var errors = [];
var missingResult = findMissingPatterns(tokenTypes);
errors = errors.concat(missingResult.errors);
var invalidResult = findInvalidPatterns(missingResult.valid);
var validTokenTypes = invalidResult.valid;
errors = errors.concat(invalidResult.errors);
errors = errors.concat(validateRegExpPattern(validTokenTypes));
errors = errors.concat(findInvalidGroupType(validTokenTypes));
errors = errors.concat(findModesThatDoNotExist(validTokenTypes, validModesNames));
errors = errors.concat(findUnreachablePatterns(validTokenTypes));
return errors;
}
exports.validatePatterns = validatePatterns;
function validateRegExpPattern(tokenTypes) {
var errors = [];
var withRegExpPatterns = (0, filter_1.default)(tokenTypes, function (currTokType) {
return (0, isRegExp_1.default)(currTokType[PATTERN]);
});
errors = errors.concat(findEndOfInputAnchor(withRegExpPatterns));
errors = errors.concat(findStartOfInputAnchor(withRegExpPatterns));
errors = errors.concat(findUnsupportedFlags(withRegExpPatterns));
errors = errors.concat(findDuplicatePatterns(withRegExpPatterns));
errors = errors.concat(findEmptyMatchRegExps(withRegExpPatterns));
return errors;
}
function findMissingPatterns(tokenTypes) {
var tokenTypesWithMissingPattern = (0, filter_1.default)(tokenTypes, function (currType) {
return !(0, has_1.default)(currType, PATTERN);
});
var errors = (0, map_1.default)(tokenTypesWithMissingPattern, function (currType) {
return {
message: "Token Type: ->" +
currType.name +
"<- missing static 'PATTERN' property",
type: lexer_public_1.LexerDefinitionErrorType.MISSING_PATTERN,
tokenTypes: [currType]
};
});
var valid = (0, difference_1.default)(tokenTypes, tokenTypesWithMissingPattern);
return { errors: errors, valid: valid };
}
exports.findMissingPatterns = findMissingPatterns;
function findInvalidPatterns(tokenTypes) {
var tokenTypesWithInvalidPattern = (0, filter_1.default)(tokenTypes, function (currType) {
var pattern = currType[PATTERN];
return (!(0, isRegExp_1.default)(pattern) &&
!(0, isFunction_1.default)(pattern) &&
!(0, has_1.default)(pattern, "exec") &&
!(0, isString_1.default)(pattern));
});
var errors = (0, map_1.default)(tokenTypesWithInvalidPattern, function (currType) {
return {
message: "Token Type: ->" +
currType.name +
"<- static 'PATTERN' can only be a RegExp, a" +
" Function matching the {CustomPatternMatcherFunc} type or an Object matching the {ICustomPattern} interface.",
type: lexer_public_1.LexerDefinitionErrorType.INVALID_PATTERN,
tokenTypes: [currType]
};
});
var valid = (0, difference_1.default)(tokenTypes, tokenTypesWithInvalidPattern);
return { errors: errors, valid: valid };
}
exports.findInvalidPatterns = findInvalidPatterns;
var end_of_input = /[^\\][$]/;
function findEndOfInputAnchor(tokenTypes) {
var EndAnchorFinder = /** @class */ (function (_super) {
__extends(EndAnchorFinder, _super);
function EndAnchorFinder() {
var _this = _super !== null && _super.apply(this, arguments) || this;
_this.found = false;
return _this;
}
EndAnchorFinder.prototype.visitEndAnchor = function (node) {
this.found = true;
};
return EndAnchorFinder;
}(regexp_to_ast_1.BaseRegExpVisitor));
var invalidRegex = (0, filter_1.default)(tokenTypes, function (currType) {
var pattern = currType.PATTERN;
try {
var regexpAst = (0, reg_exp_parser_1.getRegExpAst)(pattern);
var endAnchorVisitor = new EndAnchorFinder();
endAnchorVisitor.visit(regexpAst);
return endAnchorVisitor.found;
}
catch (e) {
// old behavior in case of runtime exceptions with regexp-to-ast.
/* istanbul ignore next - cannot ensure an error in regexp-to-ast*/
return end_of_input.test(pattern.source);
}
});
var errors = (0, map_1.default)(invalidRegex, function (currType) {
return {
message: "Unexpected RegExp Anchor Error:\n" +
"\tToken Type: ->" +
currType.name +
"<- static 'PATTERN' cannot contain end of input anchor '$'\n" +
"\tSee chevrotain.io/docs/guide/resolving_lexer_errors.html#ANCHORS" +
"\tfor details.",
type: lexer_public_1.LexerDefinitionErrorType.EOI_ANCHOR_FOUND,
tokenTypes: [currType]
};
});
return errors;
}
exports.findEndOfInputAnchor = findEndOfInputAnchor;
function findEmptyMatchRegExps(tokenTypes) {
var matchesEmptyString = (0, filter_1.default)(tokenTypes, function (currType) {
var pattern = currType.PATTERN;
return pattern.test("");
});
var errors = (0, map_1.default)(matchesEmptyString, function (currType) {
return {
message: "Token Type: ->" +
currType.name +
"<- static 'PATTERN' must not match an empty string",
type: lexer_public_1.LexerDefinitionErrorType.EMPTY_MATCH_PATTERN,
tokenTypes: [currType]
};
});
return errors;
}
exports.findEmptyMatchRegExps = findEmptyMatchRegExps;
var start_of_input = /[^\\[][\^]|^\^/;
function findStartOfInputAnchor(tokenTypes) {
var StartAnchorFinder = /** @class */ (function (_super) {
__extends(StartAnchorFinder, _super);
function StartAnchorFinder() {
var _this = _super !== null && _super.apply(this, arguments) || this;
_this.found = false;
return _this;
}
StartAnchorFinder.prototype.visitStartAnchor = function (node) {
this.found = true;
};
return StartAnchorFinder;
}(regexp_to_ast_1.BaseRegExpVisitor));
var invalidRegex = (0, filter_1.default)(tokenTypes, function (currType) {
var pattern = currType.PATTERN;
try {
var regexpAst = (0, reg_exp_parser_1.getRegExpAst)(pattern);
var startAnchorVisitor = new StartAnchorFinder();
startAnchorVisitor.visit(regexpAst);
return startAnchorVisitor.found;
}
catch (e) {
// old behavior in case of runtime exceptions with regexp-to-ast.
/* istanbul ignore next - cannot ensure an error in regexp-to-ast*/
return start_of_input.test(pattern.source);
}
});
var errors = (0, map_1.default)(invalidRegex, function (currType) {
return {
message: "Unexpected RegExp Anchor Error:\n" +
"\tToken Type: ->" +
currType.name +
"<- static 'PATTERN' cannot contain start of input anchor '^'\n" +
"\tSee https://chevrotain.io/docs/guide/resolving_lexer_errors.html#ANCHORS" +
"\tfor details.",
type: lexer_public_1.LexerDefinitionErrorType.SOI_ANCHOR_FOUND,
tokenTypes: [currType]
};
});
return errors;
}
exports.findStartOfInputAnchor = findStartOfInputAnchor;
function findUnsupportedFlags(tokenTypes) {
var invalidFlags = (0, filter_1.default)(tokenTypes, function (currType) {
var pattern = currType[PATTERN];
return pattern instanceof RegExp && (pattern.multiline || pattern.global);
});
var errors = (0, map_1.default)(invalidFlags, function (currType) {
return {
message: "Token Type: ->" +
currType.name +
"<- static 'PATTERN' may NOT contain global('g') or multiline('m')",
type: lexer_public_1.LexerDefinitionErrorType.UNSUPPORTED_FLAGS_FOUND,
tokenTypes: [currType]
};
});
return errors;
}
exports.findUnsupportedFlags = findUnsupportedFlags;
// This can only test for identical duplicate RegExps, not semantically equivalent ones.
function findDuplicatePatterns(tokenTypes) {
var found = [];
var identicalPatterns = (0, map_1.default)(tokenTypes, function (outerType) {
return (0, reduce_1.default)(tokenTypes, function (result, innerType) {
if (outerType.PATTERN.source === innerType.PATTERN.source &&
!(0, includes_1.default)(found, innerType) &&
innerType.PATTERN !== lexer_public_1.Lexer.NA) {
// this avoids duplicates in the result, each Token Type may only appear in one "set"
// in essence we are creating Equivalence classes on equality relation.
found.push(innerType);
result.push(innerType);
return result;
}
return result;
}, []);
});
identicalPatterns = (0, compact_1.default)(identicalPatterns);
var duplicatePatterns = (0, filter_1.default)(identicalPatterns, function (currIdenticalSet) {
return currIdenticalSet.length > 1;
});
var errors = (0, map_1.default)(duplicatePatterns, function (setOfIdentical) {
var tokenTypeNames = (0, map_1.default)(setOfIdentical, function (currType) {
return currType.name;
});
var dupPatternSrc = (0, first_1.default)(setOfIdentical).PATTERN;
return {
message: "The same RegExp pattern ->".concat(dupPatternSrc, "<-") +
"has been used in all of the following Token Types: ".concat(tokenTypeNames.join(", "), " <-"),
type: lexer_public_1.LexerDefinitionErrorType.DUPLICATE_PATTERNS_FOUND,
tokenTypes: setOfIdentical
};
});
return errors;
}
exports.findDuplicatePatterns = findDuplicatePatterns;
function findInvalidGroupType(tokenTypes) {
var invalidTypes = (0, filter_1.default)(tokenTypes, function (clazz) {
if (!(0, has_1.default)(clazz, "GROUP")) {
return false;
}
var group = clazz.GROUP;
return group !== lexer_public_1.Lexer.SKIPPED && group !== lexer_public_1.Lexer.NA && !(0, isString_1.default)(group);
});
var errors = (0, map_1.default)(invalidTypes, function (currType) {
return {
message: "Token Type: ->" +
currType.name +
"<- static 'GROUP' can only be Lexer.SKIPPED/Lexer.NA/A String",
type: lexer_public_1.LexerDefinitionErrorType.INVALID_GROUP_TYPE_FOUND,
tokenTypes: [currType]
};
});
return errors;
}
exports.findInvalidGroupType = findInvalidGroupType;
function findModesThatDoNotExist(tokenTypes, validModes) {
var invalidModes = (0, filter_1.default)(tokenTypes, function (clazz) {
return (clazz.PUSH_MODE !== undefined && !(0, includes_1.default)(validModes, clazz.PUSH_MODE));
});
var errors = (0, map_1.default)(invalidModes, function (tokType) {
var msg = "Token Type: ->".concat(tokType.name, "<- static 'PUSH_MODE' value cannot refer to a Lexer Mode ->").concat(tokType.PUSH_MODE, "<-") +
"which does not exist";
return {
message: msg,
type: lexer_public_1.LexerDefinitionErrorType.PUSH_MODE_DOES_NOT_EXIST,
tokenTypes: [tokType]
};
});
return errors;
}
exports.findModesThatDoNotExist = findModesThatDoNotExist;
function findUnreachablePatterns(tokenTypes) {
var errors = [];
var canBeTested = (0, reduce_1.default)(tokenTypes, function (result, tokType, idx) {
var pattern = tokType.PATTERN;
if (pattern === lexer_public_1.Lexer.NA) {
return result;
}
// a more comprehensive validation for all forms of regExps would require
// deeper regExp analysis capabilities
if ((0, isString_1.default)(pattern)) {
result.push({ str: pattern, idx: idx, tokenType: tokType });
}
else if ((0, isRegExp_1.default)(pattern) && noMetaChar(pattern)) {
result.push({ str: pattern.source, idx: idx, tokenType: tokType });
}
return result;
}, []);
(0, forEach_1.default)(tokenTypes, function (tokType, testIdx) {
(0, forEach_1.default)(canBeTested, function (_a) {
var str = _a.str, idx = _a.idx, tokenType = _a.tokenType;
if (testIdx < idx && testTokenType(str, tokType.PATTERN)) {
var msg = "Token: ->".concat(tokenType.name, "<- can never be matched.\n") +
"Because it appears AFTER the Token Type ->".concat(tokType.name, "<-") +
"in the lexer's definition.\n" +
"See https://chevrotain.io/docs/guide/resolving_lexer_errors.html#UNREACHABLE";
errors.push({
message: msg,
type: lexer_public_1.LexerDefinitionErrorType.UNREACHABLE_PATTERN,
tokenTypes: [tokType, tokenType]
});
}
});
});
return errors;
}
exports.findUnreachablePatterns = findUnreachablePatterns;
function testTokenType(str, pattern) {
/* istanbul ignore else */
if ((0, isRegExp_1.default)(pattern)) {
var regExpArray = pattern.exec(str);
return regExpArray !== null && regExpArray.index === 0;
}
else if ((0, isFunction_1.default)(pattern)) {
// maintain the API of custom patterns
return pattern(str, 0, [], {});
}
else if ((0, has_1.default)(pattern, "exec")) {
// maintain the API of custom patterns
return pattern.exec(str, 0, [], {});
}
else if (typeof pattern === "string") {
return pattern === str;
}
else {
throw Error("non exhaustive match");
}
}
function noMetaChar(regExp) {
//https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp
var metaChars = [
".",
"\\",
"[",
"]",
"|",
"^",
"$",
"(",
")",
"?",
"*",
"+",
"{"
];
return ((0, find_1.default)(metaChars, function (char) { return regExp.source.indexOf(char) !== -1; }) === undefined);
}
function addStartOfInput(pattern) {
var flags = pattern.ignoreCase ? "i" : "";
// always wrapping in a none capturing group preceded by '^' to make sure matching can only work on start of input.
// duplicate/redundant start of input markers have no meaning (/^^^^A/ === /^A/)
return new RegExp("^(?:".concat(pattern.source, ")"), flags);
}
exports.addStartOfInput = addStartOfInput;
function addStickyFlag(pattern) {
var flags = pattern.ignoreCase ? "iy" : "y";
// always wrapping in a none capturing group preceded by '^' to make sure matching can only work on start of input.
// duplicate/redundant start of input markers have no meaning (/^^^^A/ === /^A/)
return new RegExp("".concat(pattern.source), flags);
}
exports.addStickyFlag = addStickyFlag;
function performRuntimeChecks(lexerDefinition, trackLines, lineTerminatorCharacters) {
var errors = [];
// some run time checks to help the end users.
if (!(0, has_1.default)(lexerDefinition, exports.DEFAULT_MODE)) {
errors.push({
message: "A MultiMode Lexer cannot be initialized without a <" +
exports.DEFAULT_MODE +
"> property in its definition\n",
type: lexer_public_1.LexerDefinitionErrorType.MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE
});
}
if (!(0, has_1.default)(lexerDefinition, exports.MODES)) {
errors.push({
message: "A MultiMode Lexer cannot be initialized without a <" +
exports.MODES +
"> property in its definition\n",
type: lexer_public_1.LexerDefinitionErrorType.MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY
});
}
if ((0, has_1.default)(lexerDefinition, exports.MODES) &&
(0, has_1.default)(lexerDefinition, exports.DEFAULT_MODE) &&
!(0, has_1.default)(lexerDefinition.modes, lexerDefinition.defaultMode)) {
errors.push({
message: "A MultiMode Lexer cannot be initialized with a ".concat(exports.DEFAULT_MODE, ": <").concat(lexerDefinition.defaultMode, ">") +
"which does not exist\n",
type: lexer_public_1.LexerDefinitionErrorType.MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST
});
}
if ((0, has_1.default)(lexerDefinition, exports.MODES)) {
(0, forEach_1.default)(lexerDefinition.modes, function (currModeValue, currModeName) {
(0, forEach_1.default)(currModeValue, function (currTokType, currIdx) {
if ((0, isUndefined_1.default)(currTokType)) {
errors.push({
message: "A Lexer cannot be initialized using an undefined Token Type. Mode:" +
"<".concat(currModeName, "> at index: <").concat(currIdx, ">\n"),
type: lexer_public_1.LexerDefinitionErrorType.LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED
});
}
else if ((0, has_1.default)(currTokType, "LONGER_ALT")) {
var longerAlt = (0, isArray_1.default)(currTokType.LONGER_ALT)
? currTokType.LONGER_ALT
: [currTokType.LONGER_ALT];
(0, forEach_1.default)(longerAlt, function (currLongerAlt) {
if (!(0, isUndefined_1.default)(currLongerAlt) &&
!(0, includes_1.default)(currModeValue, currLongerAlt)) {
errors.push({
message: "A MultiMode Lexer cannot be initialized with a longer_alt <".concat(currLongerAlt.name, "> on token <").concat(currTokType.name, "> outside of mode <").concat(currModeName, ">\n"),
type: lexer_public_1.LexerDefinitionErrorType.MULTI_MODE_LEXER_LONGER_ALT_NOT_IN_CURRENT_MODE
});
}
});
}
});
});
}
return errors;
}
exports.performRuntimeChecks = performRuntimeChecks;
function performWarningRuntimeChecks(lexerDefinition, trackLines, lineTerminatorCharacters) {
var warnings = [];
var hasAnyLineBreak = false;
var allTokenTypes = (0, compact_1.default)((0, flatten_1.default)((0, values_1.default)(lexerDefinition.modes)));
var concreteTokenTypes = (0, reject_1.default)(allTokenTypes, function (currType) { return currType[PATTERN] === lexer_public_1.Lexer.NA; });
var terminatorCharCodes = getCharCodes(lineTerminatorCharacters);
if (trackLines) {
(0, forEach_1.default)(concreteTokenTypes, function (tokType) {
var currIssue = checkLineBreaksIssues(tokType, terminatorCharCodes);
if (currIssue !== false) {
var message = buildLineBreakIssueMessage(tokType, currIssue);
var warningDescriptor = {
message: message,
type: currIssue.issue,
tokenType: tokType
};
warnings.push(warningDescriptor);
}
else {
// we don't want to attempt to scan if the user explicitly specified the line_breaks option.
if ((0, has_1.default)(tokType, "LINE_BREAKS")) {
if (tokType.LINE_BREAKS === true) {
hasAnyLineBreak = true;
}
}
else {
if ((0, reg_exp_1.canMatchCharCode)(terminatorCharCodes, tokType.PATTERN)) {
hasAnyLineBreak = true;
}
}
}
});
}
if (trackLines && !hasAnyLineBreak) {
warnings.push({
message: "Warning: No LINE_BREAKS Found.\n" +
"\tThis Lexer has been defined to track line and column information,\n" +
"\tBut none of the Token Types can be identified as matching a line terminator.\n" +
"\tSee https://chevrotain.io/docs/guide/resolving_lexer_errors.html#LINE_BREAKS \n" +
"\tfor details.",
type: lexer_public_1.LexerDefinitionErrorType.NO_LINE_BREAKS_FLAGS
});
}
return warnings;
}
exports.performWarningRuntimeChecks = performWarningRuntimeChecks;
function cloneEmptyGroups(emptyGroups) {
var clonedResult = {};
var groupKeys = (0, keys_1.default)(emptyGroups);
(0, forEach_1.default)(groupKeys, function (currKey) {
var currGroupValue = emptyGroups[currKey];
/* istanbul ignore else */
if ((0, isArray_1.default)(currGroupValue)) {
clonedResult[currKey] = [];
}
else {
throw Error("non exhaustive match");
}
});
return clonedResult;
}
exports.cloneEmptyGroups = cloneEmptyGroups;
// TODO: refactor to avoid duplication
function isCustomPattern(tokenType) {
var pattern = tokenType.PATTERN;
/* istanbul ignore else */
if ((0, isRegExp_1.default)(pattern)) {
return false;
}
else if ((0, isFunction_1.default)(pattern)) {
// CustomPatternMatcherFunc - custom patterns do not require any transformations, only wrapping in a RegExp Like object
return true;
}
else if ((0, has_1.default)(pattern, "exec")) {
// ICustomPattern
return true;
}
else if ((0, isString_1.default)(pattern)) {
return false;
}
else {
throw Error("non exhaustive match");
}
}
exports.isCustomPattern = isCustomPattern;
function isShortPattern(pattern) {
if ((0, isString_1.default)(pattern) && pattern.length === 1) {
return pattern.charCodeAt(0);
}
else {
return false;
}
}
exports.isShortPattern = isShortPattern;
/**
* Faster than using a RegExp for default newline detection during lexing.
*/
exports.LineTerminatorOptimizedTester = {
// implements /\n|\r\n?/g.test
test: function (text) {
var len = text.length;
for (var i = this.lastIndex; i < len; i++) {
var c = text.charCodeAt(i);
if (c === 10) {
this.lastIndex = i + 1;
return true;
}
else if (c === 13) {
if (text.charCodeAt(i + 1) === 10) {
this.lastIndex = i + 2;
}
else {
this.lastIndex = i + 1;
}
return true;
}
}
return false;
},
lastIndex: 0
};
function checkLineBreaksIssues(tokType, lineTerminatorCharCodes) {
if ((0, has_1.default)(tokType, "LINE_BREAKS")) {
// if the user explicitly declared the line_breaks option we will respect their choice
// and assume it is correct.
return false;
}
else {
/* istanbul ignore else */
if ((0, isRegExp_1.default)(tokType.PATTERN)) {
try {
// TODO: why is the casting suddenly needed?
(0, reg_exp_1.canMatchCharCode)(lineTerminatorCharCodes, tokType.PATTERN);
}
catch (e) {
/* istanbul ignore next - to test this we would have to mock <canMatchCharCode> to throw an error */
return {
issue: lexer_public_1.LexerDefinitionErrorType.IDENTIFY_TERMINATOR,
errMsg: e.message
};
}
return false;
}
else if ((0, isString_1.default)(tokType.PATTERN)) {
// string literal patterns can always be analyzed to detect line terminator usage
return false;
}
else if (isCustomPattern(tokType)) {
// custom token types
return { issue: lexer_public_1.LexerDefinitionErrorType.CUSTOM_LINE_BREAK };
}
else {
throw Error("non exhaustive match");
}
}
}
function buildLineBreakIssueMessage(tokType, details) {
/* istanbul ignore else */
if (details.issue === lexer_public_1.LexerDefinitionErrorType.IDENTIFY_TERMINATOR) {
return ("Warning: unable to identify line terminator usage in pattern.\n" +
"\tThe problem is in the <".concat(tokType.name, "> Token Type\n") +
"\t Root cause: ".concat(details.errMsg, ".\n") +
"\tFor details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#IDENTIFY_TERMINATOR");
}
else if (details.issue === lexer_public_1.LexerDefinitionErrorType.CUSTOM_LINE_BREAK) {
return ("Warning: A Custom Token Pattern should specify the <line_breaks> option.\n" +
"\tThe problem is in the <".concat(tokType.name, "> Token Type\n") +
"\tFor details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#CUSTOM_LINE_BREAK");
}
else {
throw Error("non exhaustive match");
}
}
exports.buildLineBreakIssueMessage = buildLineBreakIssueMessage;
function getCharCodes(charsOrCodes) {
var charCodes = (0, map_1.default)(charsOrCodes, function (numOrString) {
if ((0, isString_1.default)(numOrString)) {
return numOrString.charCodeAt(0);
}
else {
return numOrString;
}
});
return charCodes;
}
function addToMapOfArrays(map, key, value) {
if (map[key] === undefined) {
map[key] = [value];
}
else {
map[key].push(value);
}
}
exports.minOptimizationVal = 256;
/**
* We are mapping charCode above ASCI (256) into buckets each in the size of 256.
* This is because ASCI are the most common start chars so each one of those will get its own
* possible token configs vector.
*
* Tokens starting with charCodes "above" ASCI are uncommon, so we can "afford"
* to place these into buckets of possible token configs, What we gain from
* this is avoiding the case of creating an optimization 'charCodeToPatternIdxToConfig'
* which would contain 10,000+ arrays of small size (e.g unicode Identifiers scenario).
* Our 'charCodeToPatternIdxToConfig' max size will now be:
* 256 + (2^16 / 2^8) - 1 === 511
*
* note the hack for fast division integer part extraction
* See: https://stackoverflow.com/a/4228528
*/
var charCodeToOptimizedIdxMap = [];
function charCodeToOptimizedIndex(charCode) {
return charCode < exports.minOptimizationVal
? charCode
: charCodeToOptimizedIdxMap[charCode];
}
exports.charCodeToOptimizedIndex = charCodeToOptimizedIndex;
/**
* This is a compromise between cold start / hot running performance
* Creating this array takes ~3ms on a modern machine,
* But if we perform the computation at runtime as needed the CSS Lexer benchmark
* performance degrades by ~10%
*
* TODO: Perhaps it should be lazy initialized only if a charCode > 255 is used.
*/
function initCharCodeToOptimizedIndexMap() {
if ((0, isEmpty_1.default)(charCodeToOptimizedIdxMap)) {
charCodeToOptimizedIdxMap = new Array(65536);
for (var i = 0; i < 65536; i++) {
charCodeToOptimizedIdxMap[i] = i > 255 ? 255 + ~~(i / 255) : i;
}
}
}
//# sourceMappingURL=lexer.js.map

1
_node_modules/chevrotain/lib/src/scan/lexer.js.map generated Normal file
View File

File diff suppressed because one or more lines are too long

12
_node_modules/chevrotain/lib/src/scan/lexer_errors_public.js generated Normal file
View File

@@ -0,0 +1,12 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.defaultLexerErrorProvider = void 0;
exports.defaultLexerErrorProvider = {
buildUnableToPopLexerModeMessage: function (token) {
return "Unable to pop Lexer Mode after encountering Token ->".concat(token.image, "<- The Mode Stack is empty");
},
buildUnexpectedCharactersMessage: function (fullText, startOffset, length, line, column) {
return ("unexpected character: ->".concat(fullText.charAt(startOffset), "<- at offset: ").concat(startOffset, ",") + " skipped ".concat(length, " characters."));
}
};
//# sourceMappingURL=lexer_errors_public.js.map

1
_node_modules/chevrotain/lib/src/scan/lexer_errors_public.js.map generated Normal file
View File

@@ -0,0 +1 @@
{"version":3,"file":"lexer_errors_public.js","sourceRoot":"","sources":["../../../src/scan/lexer_errors_public.ts"],"names":[],"mappings":";;;AAEa,QAAA,yBAAyB,GAA+B;IACnE,gCAAgC,YAAC,KAAa;QAC5C,OAAO,8DAAuD,KAAK,CAAC,KAAK,+BAA4B,CAAA;IACvG,CAAC;IAED,gCAAgC,YAC9B,QAAgB,EAChB,WAAmB,EACnB,MAAc,EACd,IAAa,EACb,MAAe;QAEf,OAAO,CACL,kCAA2B,QAAQ,CAAC,MAAM,CACxC,WAAW,CACZ,2BAAiB,WAAW,MAAG,GAAG,mBAAY,MAAM,iBAAc,CACpE,CAAA;IACH,CAAC;CACF,CAAA"}

669
_node_modules/chevrotain/lib/src/scan/lexer_public.js generated Normal file
View File

@@ -0,0 +1,669 @@
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Lexer = exports.LexerDefinitionErrorType = void 0;
var lexer_1 = require("./lexer");
var noop_1 = __importDefault(require("lodash/noop"));
var isEmpty_1 = __importDefault(require("lodash/isEmpty"));
var isArray_1 = __importDefault(require("lodash/isArray"));
var last_1 = __importDefault(require("lodash/last"));
var reject_1 = __importDefault(require("lodash/reject"));
var map_1 = __importDefault(require("lodash/map"));
var forEach_1 = __importDefault(require("lodash/forEach"));
var keys_1 = __importDefault(require("lodash/keys"));
var isUndefined_1 = __importDefault(require("lodash/isUndefined"));
var identity_1 = __importDefault(require("lodash/identity"));
var assign_1 = __importDefault(require("lodash/assign"));
var reduce_1 = __importDefault(require("lodash/reduce"));
var clone_1 = __importDefault(require("lodash/clone"));
var utils_1 = require("@chevrotain/utils");
var tokens_1 = require("./tokens");
var lexer_errors_public_1 = require("./lexer_errors_public");
var reg_exp_parser_1 = require("./reg_exp_parser");
var LexerDefinitionErrorType;
(function (LexerDefinitionErrorType) {
LexerDefinitionErrorType[LexerDefinitionErrorType["MISSING_PATTERN"] = 0] = "MISSING_PATTERN";
LexerDefinitionErrorType[LexerDefinitionErrorType["INVALID_PATTERN"] = 1] = "INVALID_PATTERN";
LexerDefinitionErrorType[LexerDefinitionErrorType["EOI_ANCHOR_FOUND"] = 2] = "EOI_ANCHOR_FOUND";
LexerDefinitionErrorType[LexerDefinitionErrorType["UNSUPPORTED_FLAGS_FOUND"] = 3] = "UNSUPPORTED_FLAGS_FOUND";
LexerDefinitionErrorType[LexerDefinitionErrorType["DUPLICATE_PATTERNS_FOUND"] = 4] = "DUPLICATE_PATTERNS_FOUND";
LexerDefinitionErrorType[LexerDefinitionErrorType["INVALID_GROUP_TYPE_FOUND"] = 5] = "INVALID_GROUP_TYPE_FOUND";
LexerDefinitionErrorType[LexerDefinitionErrorType["PUSH_MODE_DOES_NOT_EXIST"] = 6] = "PUSH_MODE_DOES_NOT_EXIST";
LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE"] = 7] = "MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE";
LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY"] = 8] = "MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY";
LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST"] = 9] = "MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST";
LexerDefinitionErrorType[LexerDefinitionErrorType["LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED"] = 10] = "LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED";
LexerDefinitionErrorType[LexerDefinitionErrorType["SOI_ANCHOR_FOUND"] = 11] = "SOI_ANCHOR_FOUND";
LexerDefinitionErrorType[LexerDefinitionErrorType["EMPTY_MATCH_PATTERN"] = 12] = "EMPTY_MATCH_PATTERN";
LexerDefinitionErrorType[LexerDefinitionErrorType["NO_LINE_BREAKS_FLAGS"] = 13] = "NO_LINE_BREAKS_FLAGS";
LexerDefinitionErrorType[LexerDefinitionErrorType["UNREACHABLE_PATTERN"] = 14] = "UNREACHABLE_PATTERN";
LexerDefinitionErrorType[LexerDefinitionErrorType["IDENTIFY_TERMINATOR"] = 15] = "IDENTIFY_TERMINATOR";
LexerDefinitionErrorType[LexerDefinitionErrorType["CUSTOM_LINE_BREAK"] = 16] = "CUSTOM_LINE_BREAK";
LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_LONGER_ALT_NOT_IN_CURRENT_MODE"] = 17] = "MULTI_MODE_LEXER_LONGER_ALT_NOT_IN_CURRENT_MODE";
})(LexerDefinitionErrorType = exports.LexerDefinitionErrorType || (exports.LexerDefinitionErrorType = {}));
var DEFAULT_LEXER_CONFIG = {
deferDefinitionErrorsHandling: false,
positionTracking: "full",
lineTerminatorsPattern: /\n|\r\n?/g,
lineTerminatorCharacters: ["\n", "\r"],
ensureOptimizations: false,
safeMode: false,
errorMessageProvider: lexer_errors_public_1.defaultLexerErrorProvider,
traceInitPerf: false,
skipValidations: false,
recoveryEnabled: true
};
Object.freeze(DEFAULT_LEXER_CONFIG);
var Lexer = /** @class */ (function () {
function Lexer(lexerDefinition, config) {
if (config === void 0) { config = DEFAULT_LEXER_CONFIG; }
var _this = this;
this.lexerDefinition = lexerDefinition;
this.lexerDefinitionErrors = [];
this.lexerDefinitionWarning = [];
this.patternIdxToConfig = {};
this.charCodeToPatternIdxToConfig = {};
this.modes = [];
this.emptyGroups = {};
this.trackStartLines = true;
this.trackEndLines = true;
this.hasCustom = false;
this.canModeBeOptimized = {};
// Duplicated from the parser's perf trace trait to allow future extraction
// of the lexer to a separate package.
this.TRACE_INIT = function (phaseDesc, phaseImpl) {
// No need to optimize this using NOOP pattern because
// It is not called in a hot spot...
if (_this.traceInitPerf === true) {
_this.traceInitIndent++;
var indent = new Array(_this.traceInitIndent + 1).join("\t");
if (_this.traceInitIndent < _this.traceInitMaxIdent) {
console.log("".concat(indent, "--> <").concat(phaseDesc, ">"));
}
var _a = (0, utils_1.timer)(phaseImpl), time = _a.time, value = _a.value;
/* istanbul ignore next - Difficult to reproduce specific performance behavior (>10ms) in tests */
var traceMethod = time > 10 ? console.warn : console.log;
if (_this.traceInitIndent < _this.traceInitMaxIdent) {
traceMethod("".concat(indent, "<-- <").concat(phaseDesc, "> time: ").concat(time, "ms"));
}
_this.traceInitIndent--;
return value;
}
else {
return phaseImpl();
}
};
if (typeof config === "boolean") {
throw Error("The second argument to the Lexer constructor is now an ILexerConfig Object.\n" +
"a boolean 2nd argument is no longer supported");
}
// todo: defaults func?
this.config = (0, assign_1.default)({}, DEFAULT_LEXER_CONFIG, config);
var traceInitVal = this.config.traceInitPerf;
if (traceInitVal === true) {
this.traceInitMaxIdent = Infinity;
this.traceInitPerf = true;
}
else if (typeof traceInitVal === "number") {
this.traceInitMaxIdent = traceInitVal;
this.traceInitPerf = true;
}
this.traceInitIndent = -1;
this.TRACE_INIT("Lexer Constructor", function () {
var actualDefinition;
var hasOnlySingleMode = true;
_this.TRACE_INIT("Lexer Config handling", function () {
if (_this.config.lineTerminatorsPattern ===
DEFAULT_LEXER_CONFIG.lineTerminatorsPattern) {
// optimized built-in implementation for the defaults definition of lineTerminators
_this.config.lineTerminatorsPattern = lexer_1.LineTerminatorOptimizedTester;
}
else {
if (_this.config.lineTerminatorCharacters ===
DEFAULT_LEXER_CONFIG.lineTerminatorCharacters) {
throw Error("Error: Missing <lineTerminatorCharacters> property on the Lexer config.\n" +
"\tFor details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#MISSING_LINE_TERM_CHARS");
}
}
if (config.safeMode && config.ensureOptimizations) {
throw Error('"safeMode" and "ensureOptimizations" flags are mutually exclusive.');
}
_this.trackStartLines = /full|onlyStart/i.test(_this.config.positionTracking);
_this.trackEndLines = /full/i.test(_this.config.positionTracking);
// Convert SingleModeLexerDefinition into a IMultiModeLexerDefinition.
if ((0, isArray_1.default)(lexerDefinition)) {
actualDefinition = {
modes: { defaultMode: (0, clone_1.default)(lexerDefinition) },
defaultMode: lexer_1.DEFAULT_MODE
};
}
else {
// no conversion needed, input should already be a IMultiModeLexerDefinition
hasOnlySingleMode = false;
actualDefinition = (0, clone_1.default)(lexerDefinition);
}
});
if (_this.config.skipValidations === false) {
_this.TRACE_INIT("performRuntimeChecks", function () {
_this.lexerDefinitionErrors = _this.lexerDefinitionErrors.concat((0, lexer_1.performRuntimeChecks)(actualDefinition, _this.trackStartLines, _this.config.lineTerminatorCharacters));
});
_this.TRACE_INIT("performWarningRuntimeChecks", function () {
_this.lexerDefinitionWarning = _this.lexerDefinitionWarning.concat((0, lexer_1.performWarningRuntimeChecks)(actualDefinition, _this.trackStartLines, _this.config.lineTerminatorCharacters));
});
}
// for extra robustness to avoid throwing an none informative error message
actualDefinition.modes = actualDefinition.modes
? actualDefinition.modes
: {};
// an error of undefined TokenTypes will be detected in "performRuntimeChecks" above.
// this transformation is to increase robustness in the case of partially invalid lexer definition.
(0, forEach_1.default)(actualDefinition.modes, function (currModeValue, currModeName) {
actualDefinition.modes[currModeName] = (0, reject_1.default)(currModeValue, function (currTokType) { return (0, isUndefined_1.default)(currTokType); });
});
var allModeNames = (0, keys_1.default)(actualDefinition.modes);
(0, forEach_1.default)(actualDefinition.modes, function (currModDef, currModName) {
_this.TRACE_INIT("Mode: <".concat(currModName, "> processing"), function () {
_this.modes.push(currModName);
if (_this.config.skipValidations === false) {
_this.TRACE_INIT("validatePatterns", function () {
_this.lexerDefinitionErrors = _this.lexerDefinitionErrors.concat((0, lexer_1.validatePatterns)(currModDef, allModeNames));
});
}
// If definition errors were encountered, the analysis phase may fail unexpectedly/
// Considering a lexer with definition errors may never be used, there is no point
// to performing the analysis anyhow...
if ((0, isEmpty_1.default)(_this.lexerDefinitionErrors)) {
(0, tokens_1.augmentTokenTypes)(currModDef);
var currAnalyzeResult_1;
_this.TRACE_INIT("analyzeTokenTypes", function () {
currAnalyzeResult_1 = (0, lexer_1.analyzeTokenTypes)(currModDef, {
lineTerminatorCharacters: _this.config.lineTerminatorCharacters,
positionTracking: config.positionTracking,
ensureOptimizations: config.ensureOptimizations,
safeMode: config.safeMode,
tracer: _this.TRACE_INIT
});
});
_this.patternIdxToConfig[currModName] =
currAnalyzeResult_1.patternIdxToConfig;
_this.charCodeToPatternIdxToConfig[currModName] =
currAnalyzeResult_1.charCodeToPatternIdxToConfig;
_this.emptyGroups = (0, assign_1.default)({}, _this.emptyGroups, currAnalyzeResult_1.emptyGroups);
_this.hasCustom = currAnalyzeResult_1.hasCustom || _this.hasCustom;
_this.canModeBeOptimized[currModName] =
currAnalyzeResult_1.canBeOptimized;
}
});
});
_this.defaultMode = actualDefinition.defaultMode;
if (!(0, isEmpty_1.default)(_this.lexerDefinitionErrors) &&
!_this.config.deferDefinitionErrorsHandling) {
var allErrMessages = (0, map_1.default)(_this.lexerDefinitionErrors, function (error) {
return error.message;
});
var allErrMessagesString = allErrMessages.join("-----------------------\n");
throw new Error("Errors detected in definition of Lexer:\n" + allErrMessagesString);
}
// Only print warning if there are no errors, This will avoid pl
(0, forEach_1.default)(_this.lexerDefinitionWarning, function (warningDescriptor) {
(0, utils_1.PRINT_WARNING)(warningDescriptor.message);
});
_this.TRACE_INIT("Choosing sub-methods implementations", function () {
// Choose the relevant internal implementations for this specific parser.
// These implementations should be in-lined by the JavaScript engine
// to provide optimal performance in each scenario.
if (lexer_1.SUPPORT_STICKY) {
_this.chopInput = identity_1.default;
_this.match = _this.matchWithTest;
}
else {
_this.updateLastIndex = noop_1.default;
_this.match = _this.matchWithExec;
}
if (hasOnlySingleMode) {
_this.handleModes = noop_1.default;
}
if (_this.trackStartLines === false) {
_this.computeNewColumn = identity_1.default;
}
if (_this.trackEndLines === false) {
_this.updateTokenEndLineColumnLocation = noop_1.default;
}
if (/full/i.test(_this.config.positionTracking)) {
_this.createTokenInstance = _this.createFullToken;
}
else if (/onlyStart/i.test(_this.config.positionTracking)) {
_this.createTokenInstance = _this.createStartOnlyToken;
}
else if (/onlyOffset/i.test(_this.config.positionTracking)) {
_this.createTokenInstance = _this.createOffsetOnlyToken;
}
else {
throw Error("Invalid <positionTracking> config option: \"".concat(_this.config.positionTracking, "\""));
}
if (_this.hasCustom) {
_this.addToken = _this.addTokenUsingPush;
_this.handlePayload = _this.handlePayloadWithCustom;
}
else {
_this.addToken = _this.addTokenUsingMemberAccess;
_this.handlePayload = _this.handlePayloadNoCustom;
}
});
_this.TRACE_INIT("Failed Optimization Warnings", function () {
var unOptimizedModes = (0, reduce_1.default)(_this.canModeBeOptimized, function (cannotBeOptimized, canBeOptimized, modeName) {
if (canBeOptimized === false) {
cannotBeOptimized.push(modeName);
}
return cannotBeOptimized;
}, []);
if (config.ensureOptimizations && !(0, isEmpty_1.default)(unOptimizedModes)) {
throw Error("Lexer Modes: < ".concat(unOptimizedModes.join(", "), " > cannot be optimized.\n") +
'\t Disable the "ensureOptimizations" lexer config flag to silently ignore this and run the lexer in an un-optimized mode.\n' +
"\t Or inspect the console log for details on how to resolve these issues.");
}
});
_this.TRACE_INIT("clearRegExpParserCache", function () {
(0, reg_exp_parser_1.clearRegExpParserCache)();
});
_this.TRACE_INIT("toFastProperties", function () {
(0, utils_1.toFastProperties)(_this);
});
});
}
Lexer.prototype.tokenize = function (text, initialMode) {
if (initialMode === void 0) { initialMode = this.defaultMode; }
if (!(0, isEmpty_1.default)(this.lexerDefinitionErrors)) {
var allErrMessages = (0, map_1.default)(this.lexerDefinitionErrors, function (error) {
return error.message;
});
var allErrMessagesString = allErrMessages.join("-----------------------\n");
throw new Error("Unable to Tokenize because Errors detected in definition of Lexer:\n" +
allErrMessagesString);
}
return this.tokenizeInternal(text, initialMode);
};
// There is quite a bit of duplication between this and "tokenizeInternalLazy"
// This is intentional due to performance considerations.
// this method also used quite a bit of `!` none null assertions because it is too optimized
// for `tsc` to always understand it is "safe"
Lexer.prototype.tokenizeInternal = function (text, initialMode) {
var _this = this;
var i, j, k, matchAltImage, longerAlt, matchedImage, payload, altPayload, imageLength, group, tokType, newToken, errLength, droppedChar, msg, match;
var orgText = text;
var orgLength = orgText.length;
var offset = 0;
var matchedTokensIndex = 0;
// initializing the tokensArray to the "guessed" size.
// guessing too little will still reduce the number of array re-sizes on pushes.
// guessing too large (Tested by guessing x4 too large) may cost a bit more of memory
// but would still have a faster runtime by avoiding (All but one) array resizing.
var guessedNumberOfTokens = this.hasCustom
? 0 // will break custom token pattern APIs the matchedTokens array will contain undefined elements.
: Math.floor(text.length / 10);
var matchedTokens = new Array(guessedNumberOfTokens);
var errors = [];
var line = this.trackStartLines ? 1 : undefined;
var column = this.trackStartLines ? 1 : undefined;
var groups = (0, lexer_1.cloneEmptyGroups)(this.emptyGroups);
var trackLines = this.trackStartLines;
var lineTerminatorPattern = this.config.lineTerminatorsPattern;
var currModePatternsLength = 0;
var patternIdxToConfig = [];
var currCharCodeToPatternIdxToConfig = [];
var modeStack = [];
var emptyArray = [];
Object.freeze(emptyArray);
var getPossiblePatterns;
function getPossiblePatternsSlow() {
return patternIdxToConfig;
}
function getPossiblePatternsOptimized(charCode) {
var optimizedCharIdx = (0, lexer_1.charCodeToOptimizedIndex)(charCode);
var possiblePatterns = currCharCodeToPatternIdxToConfig[optimizedCharIdx];
if (possiblePatterns === undefined) {
return emptyArray;
}
else {
return possiblePatterns;
}
}
var pop_mode = function (popToken) {
// TODO: perhaps avoid this error in the edge case there is no more input?
if (modeStack.length === 1 &&
// if we have both a POP_MODE and a PUSH_MODE this is in-fact a "transition"
// So no error should occur.
popToken.tokenType.PUSH_MODE === undefined) {
// if we try to pop the last mode there lexer will no longer have ANY mode.
// thus the pop is ignored, an error will be created and the lexer will continue parsing in the previous mode.
var msg_1 = _this.config.errorMessageProvider.buildUnableToPopLexerModeMessage(popToken);
errors.push({
offset: popToken.startOffset,
line: popToken.startLine,
column: popToken.startColumn,
length: popToken.image.length,
message: msg_1
});
}
else {
modeStack.pop();
var newMode = (0, last_1.default)(modeStack);
patternIdxToConfig = _this.patternIdxToConfig[newMode];
currCharCodeToPatternIdxToConfig =
_this.charCodeToPatternIdxToConfig[newMode];
currModePatternsLength = patternIdxToConfig.length;
var modeCanBeOptimized = _this.canModeBeOptimized[newMode] && _this.config.safeMode === false;
if (currCharCodeToPatternIdxToConfig && modeCanBeOptimized) {
getPossiblePatterns = getPossiblePatternsOptimized;
}
else {
getPossiblePatterns = getPossiblePatternsSlow;
}
}
};
function push_mode(newMode) {
modeStack.push(newMode);
currCharCodeToPatternIdxToConfig =
this.charCodeToPatternIdxToConfig[newMode];
patternIdxToConfig = this.patternIdxToConfig[newMode];
currModePatternsLength = patternIdxToConfig.length;
currModePatternsLength = patternIdxToConfig.length;
var modeCanBeOptimized = this.canModeBeOptimized[newMode] && this.config.safeMode === false;
if (currCharCodeToPatternIdxToConfig && modeCanBeOptimized) {
getPossiblePatterns = getPossiblePatternsOptimized;
}
else {
getPossiblePatterns = getPossiblePatternsSlow;
}
}
// this pattern seems to avoid a V8 de-optimization, although that de-optimization does not
// seem to matter performance wise.
push_mode.call(this, initialMode);
var currConfig;
var recoveryEnabled = this.config.recoveryEnabled;
while (offset < orgLength) {
matchedImage = null;
var nextCharCode = orgText.charCodeAt(offset);
var chosenPatternIdxToConfig = getPossiblePatterns(nextCharCode);
var chosenPatternsLength = chosenPatternIdxToConfig.length;
for (i = 0; i < chosenPatternsLength; i++) {
currConfig = chosenPatternIdxToConfig[i];
var currPattern = currConfig.pattern;
payload = null;
// manually in-lined because > 600 chars won't be in-lined in V8
var singleCharCode = currConfig.short;
if (singleCharCode !== false) {
if (nextCharCode === singleCharCode) {
// single character string
matchedImage = currPattern;
}
}
else if (currConfig.isCustom === true) {
match = currPattern.exec(orgText, offset, matchedTokens, groups);
if (match !== null) {
matchedImage = match[0];
if (match.payload !== undefined) {
payload = match.payload;
}
}
else {
matchedImage = null;
}
}
else {
this.updateLastIndex(currPattern, offset);
matchedImage = this.match(currPattern, text, offset);
}
if (matchedImage !== null) {
// even though this pattern matched we must try a another longer alternative.
// this can be used to prioritize keywords over identifiers
longerAlt = currConfig.longerAlt;
if (longerAlt !== undefined) {
// TODO: micro optimize, avoid extra prop access
// by saving/linking longerAlt on the original config?
var longerAltLength = longerAlt.length;
for (k = 0; k < longerAltLength; k++) {
var longerAltConfig = patternIdxToConfig[longerAlt[k]];
var longerAltPattern = longerAltConfig.pattern;
altPayload = null;
// single Char can never be a longer alt so no need to test it.
// manually in-lined because > 600 chars won't be in-lined in V8
if (longerAltConfig.isCustom === true) {
match = longerAltPattern.exec(orgText, offset, matchedTokens, groups);
if (match !== null) {
matchAltImage = match[0];
if (match.payload !== undefined) {
altPayload = match.payload;
}
}
else {
matchAltImage = null;
}
}
else {
this.updateLastIndex(longerAltPattern, offset);
matchAltImage = this.match(longerAltPattern, text, offset);
}
if (matchAltImage && matchAltImage.length > matchedImage.length) {
matchedImage = matchAltImage;
payload = altPayload;
currConfig = longerAltConfig;
// Exit the loop early after matching one of the longer alternatives
// The first matched alternative takes precedence
break;
}
}
}
break;
}
}
// successful match
if (matchedImage !== null) {
imageLength = matchedImage.length;
group = currConfig.group;
if (group !== undefined) {
tokType = currConfig.tokenTypeIdx;
// TODO: "offset + imageLength" and the new column may be computed twice in case of "full" location information inside
// createFullToken method
newToken = this.createTokenInstance(matchedImage, offset, tokType, currConfig.tokenType, line, column, imageLength);
this.handlePayload(newToken, payload);
// TODO: optimize NOOP in case there are no special groups?
if (group === false) {
matchedTokensIndex = this.addToken(matchedTokens, matchedTokensIndex, newToken);
}
else {
groups[group].push(newToken);
}
}
text = this.chopInput(text, imageLength);
offset = offset + imageLength;
// TODO: with newlines the column may be assigned twice
column = this.computeNewColumn(column, imageLength);
if (trackLines === true && currConfig.canLineTerminator === true) {
var numOfLTsInMatch = 0;
var foundTerminator = void 0;
var lastLTEndOffset = void 0;
lineTerminatorPattern.lastIndex = 0;
do {
foundTerminator = lineTerminatorPattern.test(matchedImage);
if (foundTerminator === true) {
lastLTEndOffset = lineTerminatorPattern.lastIndex - 1;
numOfLTsInMatch++;
}
} while (foundTerminator === true);
if (numOfLTsInMatch !== 0) {
line = line + numOfLTsInMatch;
column = imageLength - lastLTEndOffset;
this.updateTokenEndLineColumnLocation(newToken, group, lastLTEndOffset, numOfLTsInMatch, line, column, imageLength);
}
}
// will be NOOP if no modes present
this.handleModes(currConfig, pop_mode, push_mode, newToken);
}
else {
// error recovery, drop characters until we identify a valid token's start point
var errorStartOffset = offset;
var errorLine = line;
var errorColumn = column;
var foundResyncPoint = recoveryEnabled === false;
while (foundResyncPoint === false && offset < orgLength) {
// Identity Func (when sticky flag is enabled)
text = this.chopInput(text, 1);
offset++;
for (j = 0; j < currModePatternsLength; j++) {
var currConfig_1 = patternIdxToConfig[j];
var currPattern = currConfig_1.pattern;
// manually in-lined because > 600 chars won't be in-lined in V8
var singleCharCode = currConfig_1.short;
if (singleCharCode !== false) {
if (orgText.charCodeAt(offset) === singleCharCode) {
// single character string
foundResyncPoint = true;
}
}
else if (currConfig_1.isCustom === true) {
foundResyncPoint =
currPattern.exec(orgText, offset, matchedTokens, groups) !== null;
}
else {
this.updateLastIndex(currPattern, offset);
foundResyncPoint = currPattern.exec(text) !== null;
}
if (foundResyncPoint === true) {
break;
}
}
}
errLength = offset - errorStartOffset;
// at this point we either re-synced or reached the end of the input text
msg = this.config.errorMessageProvider.buildUnexpectedCharactersMessage(orgText, errorStartOffset, errLength, errorLine, errorColumn);
errors.push({
offset: errorStartOffset,
line: errorLine,
column: errorColumn,
length: errLength,
message: msg
});
if (recoveryEnabled === false) {
break;
}
}
}
// if we do have custom patterns which push directly into the
// TODO: custom tokens should not push directly??
if (!this.hasCustom) {
// if we guessed a too large size for the tokens array this will shrink it to the right size.
matchedTokens.length = matchedTokensIndex;
}
return {
tokens: matchedTokens,
groups: groups,
errors: errors
};
};
Lexer.prototype.handleModes = function (config, pop_mode, push_mode, newToken) {
if (config.pop === true) {
// need to save the PUSH_MODE property as if the mode is popped
// patternIdxToPopMode is updated to reflect the new mode after popping the stack
var pushMode = config.push;
pop_mode(newToken);
if (pushMode !== undefined) {
push_mode.call(this, pushMode);
}
}
else if (config.push !== undefined) {
push_mode.call(this, config.push);
}
};
Lexer.prototype.chopInput = function (text, length) {
return text.substring(length);
};
Lexer.prototype.updateLastIndex = function (regExp, newLastIndex) {
regExp.lastIndex = newLastIndex;
};
// TODO: decrease this under 600 characters? inspect stripping comments option in TSC compiler
Lexer.prototype.updateTokenEndLineColumnLocation = function (newToken, group, lastLTIdx, numOfLTsInMatch, line, column, imageLength) {
var lastCharIsLT, fixForEndingInLT;
if (group !== undefined) {
// a none skipped multi line Token, need to update endLine/endColumn
lastCharIsLT = lastLTIdx === imageLength - 1;
fixForEndingInLT = lastCharIsLT ? -1 : 0;
if (!(numOfLTsInMatch === 1 && lastCharIsLT === true)) {
// if a token ends in a LT that last LT only affects the line numbering of following Tokens
newToken.endLine = line + fixForEndingInLT;
// the last LT in a token does not affect the endColumn either as the [columnStart ... columnEnd)
// inclusive to exclusive range.
newToken.endColumn = column - 1 + -fixForEndingInLT;
}
// else single LT in the last character of a token, no need to modify the endLine/EndColumn
}
};
Lexer.prototype.computeNewColumn = function (oldColumn, imageLength) {
return oldColumn + imageLength;
};
Lexer.prototype.createOffsetOnlyToken = function (image, startOffset, tokenTypeIdx, tokenType) {
return {
image: image,
startOffset: startOffset,
tokenTypeIdx: tokenTypeIdx,
tokenType: tokenType
};
};
Lexer.prototype.createStartOnlyToken = function (image, startOffset, tokenTypeIdx, tokenType, startLine, startColumn) {
return {
image: image,
startOffset: startOffset,
startLine: startLine,
startColumn: startColumn,
tokenTypeIdx: tokenTypeIdx,
tokenType: tokenType
};
};
Lexer.prototype.createFullToken = function (image, startOffset, tokenTypeIdx, tokenType, startLine, startColumn, imageLength) {
return {
image: image,
startOffset: startOffset,
endOffset: startOffset + imageLength - 1,
startLine: startLine,
endLine: startLine,
startColumn: startColumn,
endColumn: startColumn + imageLength - 1,
tokenTypeIdx: tokenTypeIdx,
tokenType: tokenType
};
};
Lexer.prototype.addTokenUsingPush = function (tokenVector, index, tokenToAdd) {
tokenVector.push(tokenToAdd);
return index;
};
Lexer.prototype.addTokenUsingMemberAccess = function (tokenVector, index, tokenToAdd) {
tokenVector[index] = tokenToAdd;
index++;
return index;
};
Lexer.prototype.handlePayloadNoCustom = function (token, payload) { };
Lexer.prototype.handlePayloadWithCustom = function (token, payload) {
if (payload !== null) {
token.payload = payload;
}
};
Lexer.prototype.matchWithTest = function (pattern, text, offset) {
var found = pattern.test(text);
if (found === true) {
return text.substring(offset, pattern.lastIndex);
}
return null;
};
Lexer.prototype.matchWithExec = function (pattern, text) {
var regExpArray = pattern.exec(text);
return regExpArray !== null ? regExpArray[0] : null;
};
Lexer.SKIPPED = "This marks a skipped Token pattern, this means each token identified by it will" +
"be consumed and then thrown into oblivion, this can be used to for example to completely ignore whitespace.";
Lexer.NA = /NOT_APPLICABLE/;
return Lexer;
}());
exports.Lexer = Lexer;
//# sourceMappingURL=lexer_public.js.map

1
_node_modules/chevrotain/lib/src/scan/lexer_public.js.map generated Normal file
View File

File diff suppressed because one or more lines are too long

273
_node_modules/chevrotain/lib/src/scan/reg_exp.js generated Normal file
View File

@@ -0,0 +1,273 @@
"use strict";
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
if (typeof b !== "function" && b !== null)
throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.canMatchCharCode = exports.firstCharOptimizedIndices = exports.getOptimizedStartCodesIndices = exports.failedOptimizationPrefixMsg = void 0;
var regexp_to_ast_1 = require("regexp-to-ast");
var isArray_1 = __importDefault(require("lodash/isArray"));
var every_1 = __importDefault(require("lodash/every"));
var forEach_1 = __importDefault(require("lodash/forEach"));
var find_1 = __importDefault(require("lodash/find"));
var values_1 = __importDefault(require("lodash/values"));
var includes_1 = __importDefault(require("lodash/includes"));
var utils_1 = require("@chevrotain/utils");
var reg_exp_parser_1 = require("./reg_exp_parser");
var lexer_1 = require("./lexer");
var complementErrorMessage = "Complement Sets are not supported for first char optimization";
exports.failedOptimizationPrefixMsg = 'Unable to use "first char" lexer optimizations:\n';
function getOptimizedStartCodesIndices(regExp, ensureOptimizations) {
if (ensureOptimizations === void 0) { ensureOptimizations = false; }
try {
var ast = (0, reg_exp_parser_1.getRegExpAst)(regExp);
var firstChars = firstCharOptimizedIndices(ast.value, {}, ast.flags.ignoreCase);
return firstChars;
}
catch (e) {
/* istanbul ignore next */
// Testing this relies on the regexp-to-ast library having a bug... */
// TODO: only the else branch needs to be ignored, try to fix with newer prettier / tsc
if (e.message === complementErrorMessage) {
if (ensureOptimizations) {
(0, utils_1.PRINT_WARNING)("".concat(exports.failedOptimizationPrefixMsg) +
"\tUnable to optimize: < ".concat(regExp.toString(), " >\n") +
"\tComplement Sets cannot be automatically optimized.\n" +
"\tThis will disable the lexer's first char optimizations.\n" +
"\tSee: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#COMPLEMENT for details.");
}
}
else {
var msgSuffix = "";
if (ensureOptimizations) {
msgSuffix =
"\n\tThis will disable the lexer's first char optimizations.\n" +
"\tSee: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#REGEXP_PARSING for details.";
}
(0, utils_1.PRINT_ERROR)("".concat(exports.failedOptimizationPrefixMsg, "\n") +
"\tFailed parsing: < ".concat(regExp.toString(), " >\n") +
"\tUsing the regexp-to-ast library version: ".concat(regexp_to_ast_1.VERSION, "\n") +
"\tPlease open an issue at: https://github.com/bd82/regexp-to-ast/issues" +
msgSuffix);
}
}
return [];
}
exports.getOptimizedStartCodesIndices = getOptimizedStartCodesIndices;
function firstCharOptimizedIndices(ast, result, ignoreCase) {
switch (ast.type) {
case "Disjunction":
for (var i = 0; i < ast.value.length; i++) {
firstCharOptimizedIndices(ast.value[i], result, ignoreCase);
}
break;
case "Alternative":
var terms = ast.value;
for (var i = 0; i < terms.length; i++) {
var term = terms[i];
// skip terms that cannot effect the first char results
switch (term.type) {
case "EndAnchor":
// A group back reference cannot affect potential starting char.
// because if a back reference is the first production than automatically
// the group being referenced has had to come BEFORE so its codes have already been added
case "GroupBackReference":
// assertions do not affect potential starting codes
case "Lookahead":
case "NegativeLookahead":
case "StartAnchor":
case "WordBoundary":
case "NonWordBoundary":
continue;
}
var atom = term;
switch (atom.type) {
case "Character":
addOptimizedIdxToResult(atom.value, result, ignoreCase);
break;
case "Set":
if (atom.complement === true) {
throw Error(complementErrorMessage);
}
(0, forEach_1.default)(atom.value, function (code) {
if (typeof code === "number") {
addOptimizedIdxToResult(code, result, ignoreCase);
}
else {
// range
var range = code;
// cannot optimize when ignoreCase is
if (ignoreCase === true) {
for (var rangeCode = range.from; rangeCode <= range.to; rangeCode++) {
addOptimizedIdxToResult(rangeCode, result, ignoreCase);
}
}
// Optimization (2 orders of magnitude less work for very large ranges)
else {
// handle unoptimized values
for (var rangeCode = range.from; rangeCode <= range.to && rangeCode < lexer_1.minOptimizationVal; rangeCode++) {
addOptimizedIdxToResult(rangeCode, result, ignoreCase);
}
// Less common charCode where we optimize for faster init time, by using larger "buckets"
if (range.to >= lexer_1.minOptimizationVal) {
var minUnOptVal = range.from >= lexer_1.minOptimizationVal
? range.from
: lexer_1.minOptimizationVal;
var maxUnOptVal = range.to;
var minOptIdx = (0, lexer_1.charCodeToOptimizedIndex)(minUnOptVal);
var maxOptIdx = (0, lexer_1.charCodeToOptimizedIndex)(maxUnOptVal);
for (var currOptIdx = minOptIdx; currOptIdx <= maxOptIdx; currOptIdx++) {
result[currOptIdx] = currOptIdx;
}
}
}
}
});
break;
case "Group":
firstCharOptimizedIndices(atom.value, result, ignoreCase);
break;
/* istanbul ignore next */
default:
throw Error("Non Exhaustive Match");
}
// reached a mandatory production, no more **start** codes can be found on this alternative
var isOptionalQuantifier = atom.quantifier !== undefined && atom.quantifier.atLeast === 0;
if (
// A group may be optional due to empty contents /(?:)/
// or if everything inside it is optional /((a)?)/
(atom.type === "Group" && isWholeOptional(atom) === false) ||
// If this term is not a group it may only be optional if it has an optional quantifier
(atom.type !== "Group" && isOptionalQuantifier === false)) {
break;
}
}
break;
/* istanbul ignore next */
default:
throw Error("non exhaustive match!");
}
// console.log(Object.keys(result).length)
return (0, values_1.default)(result);
}
exports.firstCharOptimizedIndices = firstCharOptimizedIndices;
function addOptimizedIdxToResult(code, result, ignoreCase) {
var optimizedCharIdx = (0, lexer_1.charCodeToOptimizedIndex)(code);
result[optimizedCharIdx] = optimizedCharIdx;
if (ignoreCase === true) {
handleIgnoreCase(code, result);
}
}
function handleIgnoreCase(code, result) {
var char = String.fromCharCode(code);
var upperChar = char.toUpperCase();
/* istanbul ignore else */
if (upperChar !== char) {
var optimizedCharIdx = (0, lexer_1.charCodeToOptimizedIndex)(upperChar.charCodeAt(0));
result[optimizedCharIdx] = optimizedCharIdx;
}
else {
var lowerChar = char.toLowerCase();
if (lowerChar !== char) {
var optimizedCharIdx = (0, lexer_1.charCodeToOptimizedIndex)(lowerChar.charCodeAt(0));
result[optimizedCharIdx] = optimizedCharIdx;
}
}
}
function findCode(setNode, targetCharCodes) {
return (0, find_1.default)(setNode.value, function (codeOrRange) {
if (typeof codeOrRange === "number") {
return (0, includes_1.default)(targetCharCodes, codeOrRange);
}
else {
// range
var range_1 = codeOrRange;
return ((0, find_1.default)(targetCharCodes, function (targetCode) { return range_1.from <= targetCode && targetCode <= range_1.to; }) !== undefined);
}
});
}
function isWholeOptional(ast) {
var quantifier = ast.quantifier;
if (quantifier && quantifier.atLeast === 0) {
return true;
}
if (!ast.value) {
return false;
}
return (0, isArray_1.default)(ast.value)
? (0, every_1.default)(ast.value, isWholeOptional)
: isWholeOptional(ast.value);
}
var CharCodeFinder = /** @class */ (function (_super) {
__extends(CharCodeFinder, _super);
function CharCodeFinder(targetCharCodes) {
var _this = _super.call(this) || this;
_this.targetCharCodes = targetCharCodes;
_this.found = false;
return _this;
}
CharCodeFinder.prototype.visitChildren = function (node) {
// No need to keep looking...
if (this.found === true) {
return;
}
// switch lookaheads as they do not actually consume any characters thus
// finding a charCode at lookahead context does not mean that regexp can actually contain it in a match.
switch (node.type) {
case "Lookahead":
this.visitLookahead(node);
return;
case "NegativeLookahead":
this.visitNegativeLookahead(node);
return;
}
_super.prototype.visitChildren.call(this, node);
};
CharCodeFinder.prototype.visitCharacter = function (node) {
if ((0, includes_1.default)(this.targetCharCodes, node.value)) {
this.found = true;
}
};
CharCodeFinder.prototype.visitSet = function (node) {
if (node.complement) {
if (findCode(node, this.targetCharCodes) === undefined) {
this.found = true;
}
}
else {
if (findCode(node, this.targetCharCodes) !== undefined) {
this.found = true;
}
}
};
return CharCodeFinder;
}(regexp_to_ast_1.BaseRegExpVisitor));
function canMatchCharCode(charCodes, pattern) {
if (pattern instanceof RegExp) {
var ast = (0, reg_exp_parser_1.getRegExpAst)(pattern);
var charCodeFinder = new CharCodeFinder(charCodes);
charCodeFinder.visit(ast);
return charCodeFinder.found;
}
else {
return ((0, find_1.default)(pattern, function (char) {
return (0, includes_1.default)(charCodes, char.charCodeAt(0));
}) !== undefined);
}
}
exports.canMatchCharCode = canMatchCharCode;
//# sourceMappingURL=reg_exp.js.map

1
_node_modules/chevrotain/lib/src/scan/reg_exp.js.map generated Normal file
View File

File diff suppressed because one or more lines are too long

23
_node_modules/chevrotain/lib/src/scan/reg_exp_parser.js generated Normal file
View File

@@ -0,0 +1,23 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.clearRegExpParserCache = exports.getRegExpAst = void 0;
var regexp_to_ast_1 = require("regexp-to-ast");
var regExpAstCache = {};
var regExpParser = new regexp_to_ast_1.RegExpParser();
function getRegExpAst(regExp) {
var regExpStr = regExp.toString();
if (regExpAstCache.hasOwnProperty(regExpStr)) {
return regExpAstCache[regExpStr];
}
else {
var regExpAst = regExpParser.pattern(regExpStr);
regExpAstCache[regExpStr] = regExpAst;
return regExpAst;
}
}
exports.getRegExpAst = getRegExpAst;
function clearRegExpParserCache() {
regExpAstCache = {};
}
exports.clearRegExpParserCache = clearRegExpParserCache;
//# sourceMappingURL=reg_exp_parser.js.map

1
_node_modules/chevrotain/lib/src/scan/reg_exp_parser.js.map generated Normal file
View File

@@ -0,0 +1 @@
{"version":3,"file":"reg_exp_parser.js","sourceRoot":"","sources":["../../../src/scan/reg_exp_parser.ts"],"names":[],"mappings":";;;AAAA,+CAOsB;AAEtB,IAAI,cAAc,GAAuC,EAAE,CAAA;AAC3D,IAAM,YAAY,GAAG,IAAI,4BAAY,EAAE,CAAA;AAUvC,SAAgB,YAAY,CAAC,MAAc;IACzC,IAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;IACnC,IAAI,cAAc,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE;QAC5C,OAAO,cAAc,CAAC,SAAS,CAAC,CAAA;KACjC;SAAM;QACL,IAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,cAAc,CAAC,SAAS,CAAC,GAAG,SAAS,CAAA;QACrC,OAAO,SAAS,CAAA;KACjB;AACH,CAAC;AATD,oCASC;AAED,SAAgB,sBAAsB;IACpC,cAAc,GAAG,EAAE,CAAA;AACrB,CAAC;AAFD,wDAEC"}

142
_node_modules/chevrotain/lib/src/scan/tokens.js generated Normal file
View File

@@ -0,0 +1,142 @@
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.isTokenType = exports.hasExtendingTokensTypesMapProperty = exports.hasExtendingTokensTypesProperty = exports.hasCategoriesProperty = exports.hasShortKeyProperty = exports.singleAssignCategoriesToksMap = exports.assignCategoriesMapProp = exports.assignCategoriesTokensProp = exports.assignTokenDefaultProps = exports.expandCategories = exports.augmentTokenTypes = exports.tokenIdxToClass = exports.tokenShortNameIdx = exports.tokenStructuredMatcherNoCategories = exports.tokenStructuredMatcher = void 0;
var isEmpty_1 = __importDefault(require("lodash/isEmpty"));
var compact_1 = __importDefault(require("lodash/compact"));
var isArray_1 = __importDefault(require("lodash/isArray"));
var flatten_1 = __importDefault(require("lodash/flatten"));
var difference_1 = __importDefault(require("lodash/difference"));
var map_1 = __importDefault(require("lodash/map"));
var forEach_1 = __importDefault(require("lodash/forEach"));
var has_1 = __importDefault(require("lodash/has"));
var includes_1 = __importDefault(require("lodash/includes"));
var clone_1 = __importDefault(require("lodash/clone"));
function tokenStructuredMatcher(tokInstance, tokConstructor) {
var instanceType = tokInstance.tokenTypeIdx;
if (instanceType === tokConstructor.tokenTypeIdx) {
return true;
}
else {
return (tokConstructor.isParent === true &&
tokConstructor.categoryMatchesMap[instanceType] === true);
}
}
exports.tokenStructuredMatcher = tokenStructuredMatcher;
// Optimized tokenMatcher in case our grammar does not use token categories
// Being so tiny it is much more likely to be in-lined and this avoid the function call overhead
function tokenStructuredMatcherNoCategories(token, tokType) {
return token.tokenTypeIdx === tokType.tokenTypeIdx;
}
exports.tokenStructuredMatcherNoCategories = tokenStructuredMatcherNoCategories;
exports.tokenShortNameIdx = 1;
exports.tokenIdxToClass = {};
function augmentTokenTypes(tokenTypes) {
// collect the parent Token Types as well.
var tokenTypesAndParents = expandCategories(tokenTypes);
// add required tokenType and categoryMatches properties
assignTokenDefaultProps(tokenTypesAndParents);
// fill up the categoryMatches
assignCategoriesMapProp(tokenTypesAndParents);
assignCategoriesTokensProp(tokenTypesAndParents);
(0, forEach_1.default)(tokenTypesAndParents, function (tokType) {
tokType.isParent = tokType.categoryMatches.length > 0;
});
}
exports.augmentTokenTypes = augmentTokenTypes;
function expandCategories(tokenTypes) {
var result = (0, clone_1.default)(tokenTypes);
var categories = tokenTypes;
var searching = true;
while (searching) {
categories = (0, compact_1.default)((0, flatten_1.default)((0, map_1.default)(categories, function (currTokType) { return currTokType.CATEGORIES; })));
var newCategories = (0, difference_1.default)(categories, result);
result = result.concat(newCategories);
if ((0, isEmpty_1.default)(newCategories)) {
searching = false;
}
else {
categories = newCategories;
}
}
return result;
}
exports.expandCategories = expandCategories;
function assignTokenDefaultProps(tokenTypes) {
(0, forEach_1.default)(tokenTypes, function (currTokType) {
if (!hasShortKeyProperty(currTokType)) {
exports.tokenIdxToClass[exports.tokenShortNameIdx] = currTokType;
currTokType.tokenTypeIdx = exports.tokenShortNameIdx++;
}
// CATEGORIES? : TokenType | TokenType[]
if (hasCategoriesProperty(currTokType) &&
!(0, isArray_1.default)(currTokType.CATEGORIES)
// &&
// !isUndefined(currTokType.CATEGORIES.PATTERN)
) {
currTokType.CATEGORIES = [currTokType.CATEGORIES];
}
if (!hasCategoriesProperty(currTokType)) {
currTokType.CATEGORIES = [];
}
if (!hasExtendingTokensTypesProperty(currTokType)) {
currTokType.categoryMatches = [];
}
if (!hasExtendingTokensTypesMapProperty(currTokType)) {
currTokType.categoryMatchesMap = {};
}
});
}
exports.assignTokenDefaultProps = assignTokenDefaultProps;
function assignCategoriesTokensProp(tokenTypes) {
(0, forEach_1.default)(tokenTypes, function (currTokType) {
// avoid duplications
currTokType.categoryMatches = [];
(0, forEach_1.default)(currTokType.categoryMatchesMap, function (val, key) {
currTokType.categoryMatches.push(exports.tokenIdxToClass[key].tokenTypeIdx);
});
});
}
exports.assignCategoriesTokensProp = assignCategoriesTokensProp;
function assignCategoriesMapProp(tokenTypes) {
(0, forEach_1.default)(tokenTypes, function (currTokType) {
singleAssignCategoriesToksMap([], currTokType);
});
}
exports.assignCategoriesMapProp = assignCategoriesMapProp;
function singleAssignCategoriesToksMap(path, nextNode) {
(0, forEach_1.default)(path, function (pathNode) {
nextNode.categoryMatchesMap[pathNode.tokenTypeIdx] = true;
});
(0, forEach_1.default)(nextNode.CATEGORIES, function (nextCategory) {
var newPath = path.concat(nextNode);
// avoids infinite loops due to cyclic categories.
if (!(0, includes_1.default)(newPath, nextCategory)) {
singleAssignCategoriesToksMap(newPath, nextCategory);
}
});
}
exports.singleAssignCategoriesToksMap = singleAssignCategoriesToksMap;
function hasShortKeyProperty(tokType) {
return (0, has_1.default)(tokType, "tokenTypeIdx");
}
exports.hasShortKeyProperty = hasShortKeyProperty;
function hasCategoriesProperty(tokType) {
return (0, has_1.default)(tokType, "CATEGORIES");
}
exports.hasCategoriesProperty = hasCategoriesProperty;
function hasExtendingTokensTypesProperty(tokType) {
return (0, has_1.default)(tokType, "categoryMatches");
}
exports.hasExtendingTokensTypesProperty = hasExtendingTokensTypesProperty;
function hasExtendingTokensTypesMapProperty(tokType) {
return (0, has_1.default)(tokType, "categoryMatchesMap");
}
exports.hasExtendingTokensTypesMapProperty = hasExtendingTokensTypesMapProperty;
function isTokenType(tokType) {
return (0, has_1.default)(tokType, "tokenTypeIdx");
}
exports.isTokenType = isTokenType;
//# sourceMappingURL=tokens.js.map

1
_node_modules/chevrotain/lib/src/scan/tokens.js.map generated Normal file
View File

@@ -0,0 +1 @@
{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../../src/scan/tokens.ts"],"names":[],"mappings":";;;;;;AAAA,2DAAoC;AACpC,2DAAoC;AACpC,2DAAoC;AACpC,2DAAoC;AACpC,iEAA0C;AAC1C,mDAA4B;AAC5B,2DAAoC;AACpC,mDAA4B;AAC5B,6DAAsC;AACtC,uDAAgC;AAGhC,SAAgB,sBAAsB,CACpC,WAAmB,EACnB,cAAyB;IAEzB,IAAM,YAAY,GAAG,WAAW,CAAC,YAAY,CAAA;IAC7C,IAAI,YAAY,KAAK,cAAc,CAAC,YAAY,EAAE;QAChD,OAAO,IAAI,CAAA;KACZ;SAAM;QACL,OAAO,CACL,cAAc,CAAC,QAAQ,KAAK,IAAI;YAChC,cAAc,CAAC,kBAAmB,CAAC,YAAY,CAAC,KAAK,IAAI,CAC1D,CAAA;KACF;AACH,CAAC;AAbD,wDAaC;AAED,2EAA2E;AAC3E,gGAAgG;AAChG,SAAgB,kCAAkC,CAChD,KAAa,EACb,OAAkB;IAElB,OAAO,KAAK,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,CAAA;AACpD,CAAC;AALD,gFAKC;AAEU,QAAA,iBAAiB,GAAG,CAAC,CAAA;AACnB,QAAA,eAAe,GAAsC,EAAE,CAAA;AAEpE,SAAgB,iBAAiB,CAAC,UAAuB;IACvD,0CAA0C;IAC1C,IAAM,oBAAoB,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAA;IAEzD,wDAAwD;IACxD,uBAAuB,CAAC,oBAAoB,CAAC,CAAA;IAE7C,8BAA8B;IAC9B,uBAAuB,CAAC,oBAAoB,CAAC,CAAA;IAC7C,0BAA0B,CAAC,oBAAoB,CAAC,CAAA;IAEhD,IAAA,iBAAO,EAAC,oBAAoB,EAAE,UAAC,OAAO;QACpC,OAAO,CAAC,QAAQ,GAAG,OAAO,CAAC,eAAgB,CAAC,MAAM,GAAG,CAAC,CAAA;IACxD,CAAC,CAAC,CAAA;AACJ,CAAC;AAdD,8CAcC;AAED,SAAgB,gBAAgB,CAAC,UAAuB;IACtD,IAAI,MAAM,GAAG,IAAA,eAAK,EAAC,UAAU,CAAC,CAAA;IAE9B,IAAI,UAAU,GAAG,UAAU,CAAA;IAC3B,IAAI,SAAS,GAAG,IAAI,CAAA;IACpB,OAAO,SAAS,EAAE;QAChB,UAAU,GAAG,IAAA,iBAAO,EAClB,IAAA,iBAAO,EAAC,IAAA,aAAG,EAAC,UAAU,EAAE,UAAC,WAAW,IAAK,OAAA,WAAW,CAAC,UAAU,EAAtB,CAAsB,CAAC,CAAC,CAClE,CAAA;QAED,IAAM,aAAa,GAAG,IAAA,oBAAU,EAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QAEpD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAErC,IAAI,IAAA,iBAAO,EAAC,aAAa,CAAC,EAAE;YAC1B,SAAS,GAAG,KAAK,CAAA;SAClB;aAAM;YACL,UAAU,GAAG,aAAa,CAAA;SAC3B;KACF;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AArBD,4CAqBC;AAED,SAAgB,uBAAuB,CAAC,UAAuB;IAC7D,IAAA,iBAAO,EAAC,UAAU,EAAE,UAAC,WAAW;QAC9B,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE;YACrC,uBAAe,CAAC,yBAAiB,CAAC,GAAG,WAAW,CAC/C;YAAM,WAAY,CAAC,YAAY,GAAG,yBAAiB,EAAE,CAAA;SACvD;QAED,wCAAwC;QACxC,IACE,qBAAqB,CAAC,WAAW,CAAC;YAClC,CAAC,IAAA,iBAAO,EAAC,WAAW,CAAC,UAAU,CAAC;QAChC,KAAK;QACL,+CAA+C;UAC/C;YACA,WAAW,CAAC,UAAU,GAAG,CAAC,WAAW,CAAC,UAAkC,CAAC,CAAA;SAC1E;QAED,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,EAAE;YACvC,WAAW,CAAC,UAAU,GAAG,EAAE,CAAA;SAC5B;QAED,IAAI,CAAC,+BAA+B,CAAC,WAAW,CAAC,EAAE;YACjD,WAAW,CAAC,eAAe,GAAG,EAAE,CAAA;SACjC;QAED,IAAI,CAAC,kCAAkC,CAAC,WAAW,CAAC,EAAE;YACpD,WAAW,CAAC,kBAAkB,GAAG,EAAE,CAAA;SACpC;IACH,CAAC,CAAC,CAAA;AACJ,CAAC;AA7BD,0DA6BC;AAED,SAAgB,0BAA0B,CAAC,UAAuB;IAChE,IAAA,iBAAO,EAAC,UAAU,EAAE,UAAC,WAAW;QAC9B,qBAAqB;QACrB,WAAW,CAAC,eAAe,GAAG,EAAE,CAAA;QAChC,IAAA,iBAAO,EAAC,WAAW,CAAC,kBAAmB,EAAE,UAAC,GAAG,EAAE,GAAG;YAChD,WAAW,CAAC,eAAgB,CAAC,IAAI,CAC/B,uBAAe,CAAC,GAAwB,CAAC,CAAC,YAAa,CACxD,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC;AAVD,gEAUC;AAED,SAAgB,uBAAuB,CAAC,UAAuB;IAC7D,IAAA,iBAAO,EAAC,UAAU,EAAE,UAAC,WAAW;QAC9B,6BAA6B,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;IAChD,CAAC,CAAC,CAAA;AACJ,CAAC;AAJD,0DAIC;AAED,SAAgB,6BAA6B,CAC3C,IAAiB,EACjB,QAAmB;IAEnB,IAAA,iBAAO,EAAC,IAAI,EAAE,UAAC,QAAQ;QACrB,QAAQ,CAAC,kBAAmB,CAAC,QAAQ,CAAC,YAAa,CAAC,GAAG,IAAI,CAAA;IAC7D,CAAC,CAAC,CAAA;IAEF,IAAA,iBAAO,EAAC,QAAQ,CAAC,UAAU,EAAE,UAAC,YAAY;QACxC,IAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QACrC,kDAAkD;QAClD,IAAI,CAAC,IAAA,kBAAQ,EAAC,OAAO,EAAE,YAAY,CAAC,EAAE;YACpC,6BAA6B,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;SACrD;IACH,CAAC,CAAC,CAAA;AACJ,CAAC;AAfD,sEAeC;AAED,SAAgB,mBAAmB,CAAC,OAAkB;IACpD,OAAO,IAAA,aAAG,EAAC,OAAO,EAAE,cAAc,CAAC,CAAA;AACrC,CAAC;AAFD,kDAEC;AAED,SAAgB,qBAAqB,CAAC,OAAkB;IACtD,OAAO,IAAA,aAAG,EAAC,OAAO,EAAE,YAAY,CAAC,CAAA;AACnC,CAAC;AAFD,sDAEC;AAED,SAAgB,+BAA+B,CAAC,OAAkB;IAChE,OAAO,IAAA,aAAG,EAAC,OAAO,EAAE,iBAAiB,CAAC,CAAA;AACxC,CAAC;AAFD,0EAEC;AAED,SAAgB,kCAAkC,CAChD,OAAkB;IAElB,OAAO,IAAA,aAAG,EAAC,OAAO,EAAE,oBAAoB,CAAC,CAAA;AAC3C,CAAC;AAJD,gFAIC;AAED,SAAgB,WAAW,CAAC,OAAkB;IAC5C,OAAO,IAAA,aAAG,EAAC,OAAO,EAAE,cAAc,CAAC,CAAA;AACrC,CAAC;AAFD,kCAEC"}

5
_node_modules/chevrotain/lib/src/scan/tokens_constants.js generated Normal file
View File

@@ -0,0 +1,5 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.EOF_TOKEN_TYPE = void 0;
exports.EOF_TOKEN_TYPE = 1;
//# sourceMappingURL=tokens_constants.js.map

1
_node_modules/chevrotain/lib/src/scan/tokens_constants.js.map generated Normal file
View File

@@ -0,0 +1 @@
{"version":3,"file":"tokens_constants.js","sourceRoot":"","sources":["../../../src/scan/tokens_constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,cAAc,GAAG,CAAC,CAAA"}

101
_node_modules/chevrotain/lib/src/scan/tokens_public.js generated Normal file
View File

@@ -0,0 +1,101 @@
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.tokenMatcher = exports.createTokenInstance = exports.EOF = exports.createToken = exports.hasTokenLabel = exports.tokenName = exports.tokenLabel = void 0;
var isString_1 = __importDefault(require("lodash/isString"));
var has_1 = __importDefault(require("lodash/has"));
var isUndefined_1 = __importDefault(require("lodash/isUndefined"));
var lexer_public_1 = require("./lexer_public");
var tokens_1 = require("./tokens");
function tokenLabel(tokType) {
if (hasTokenLabel(tokType)) {
return tokType.LABEL;
}
else {
return tokType.name;
}
}
exports.tokenLabel = tokenLabel;
function tokenName(tokType) {
return tokType.name;
}
exports.tokenName = tokenName;
function hasTokenLabel(obj) {
return (0, isString_1.default)(obj.LABEL) && obj.LABEL !== "";
}
exports.hasTokenLabel = hasTokenLabel;
var PARENT = "parent";
var CATEGORIES = "categories";
var LABEL = "label";
var GROUP = "group";
var PUSH_MODE = "push_mode";
var POP_MODE = "pop_mode";
var LONGER_ALT = "longer_alt";
var LINE_BREAKS = "line_breaks";
var START_CHARS_HINT = "start_chars_hint";
function createToken(config) {
return createTokenInternal(config);
}
exports.createToken = createToken;
function createTokenInternal(config) {
var pattern = config.pattern;
var tokenType = {};
tokenType.name = config.name;
if (!(0, isUndefined_1.default)(pattern)) {
tokenType.PATTERN = pattern;
}
if ((0, has_1.default)(config, PARENT)) {
throw ("The parent property is no longer supported.\n" +
"See: https://github.com/chevrotain/chevrotain/issues/564#issuecomment-349062346 for details.");
}
if ((0, has_1.default)(config, CATEGORIES)) {
// casting to ANY as this will be fixed inside `augmentTokenTypes``
tokenType.CATEGORIES = config[CATEGORIES];
}
(0, tokens_1.augmentTokenTypes)([tokenType]);
if ((0, has_1.default)(config, LABEL)) {
tokenType.LABEL = config[LABEL];
}
if ((0, has_1.default)(config, GROUP)) {
tokenType.GROUP = config[GROUP];
}
if ((0, has_1.default)(config, POP_MODE)) {
tokenType.POP_MODE = config[POP_MODE];
}
if ((0, has_1.default)(config, PUSH_MODE)) {
tokenType.PUSH_MODE = config[PUSH_MODE];
}
if ((0, has_1.default)(config, LONGER_ALT)) {
tokenType.LONGER_ALT = config[LONGER_ALT];
}
if ((0, has_1.default)(config, LINE_BREAKS)) {
tokenType.LINE_BREAKS = config[LINE_BREAKS];
}
if ((0, has_1.default)(config, START_CHARS_HINT)) {
tokenType.START_CHARS_HINT = config[START_CHARS_HINT];
}
return tokenType;
}
exports.EOF = createToken({ name: "EOF", pattern: lexer_public_1.Lexer.NA });
(0, tokens_1.augmentTokenTypes)([exports.EOF]);
function createTokenInstance(tokType, image, startOffset, endOffset, startLine, endLine, startColumn, endColumn) {
return {
image: image,
startOffset: startOffset,
endOffset: endOffset,
startLine: startLine,
endLine: endLine,
startColumn: startColumn,
endColumn: endColumn,
tokenTypeIdx: tokType.tokenTypeIdx,
tokenType: tokType
};
}
exports.createTokenInstance = createTokenInstance;
function tokenMatcher(token, tokType) {
return (0, tokens_1.tokenStructuredMatcher)(token, tokType);
}
exports.tokenMatcher = tokenMatcher;
//# sourceMappingURL=tokens_public.js.map

1
_node_modules/chevrotain/lib/src/scan/tokens_public.js.map generated Normal file
View File

@@ -0,0 +1 @@
{"version":3,"file":"tokens_public.js","sourceRoot":"","sources":["../../../src/scan/tokens_public.ts"],"names":[],"mappings":";;;;;;AAAA,6DAAsC;AACtC,mDAA4B;AAC5B,mEAA4C;AAC5C,+CAAsC;AACtC,mCAAoE;AAGpE,SAAgB,UAAU,CAAC,OAAkB;IAC3C,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE;QAC1B,OAAO,OAAO,CAAC,KAAK,CAAA;KACrB;SAAM;QACL,OAAO,OAAO,CAAC,IAAI,CAAA;KACpB;AACH,CAAC;AAND,gCAMC;AAED,SAAgB,SAAS,CAAC,OAAkB;IAC1C,OAAO,OAAO,CAAC,IAAI,CAAA;AACrB,CAAC;AAFD,8BAEC;AAED,SAAgB,aAAa,CAC3B,GAAc;IAEd,OAAO,IAAA,kBAAQ,EAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,KAAK,KAAK,EAAE,CAAA;AAChD,CAAC;AAJD,sCAIC;AAED,IAAM,MAAM,GAAG,QAAQ,CAAA;AACvB,IAAM,UAAU,GAAG,YAAY,CAAA;AAC/B,IAAM,KAAK,GAAG,OAAO,CAAA;AACrB,IAAM,KAAK,GAAG,OAAO,CAAA;AACrB,IAAM,SAAS,GAAG,WAAW,CAAA;AAC7B,IAAM,QAAQ,GAAG,UAAU,CAAA;AAC3B,IAAM,UAAU,GAAG,YAAY,CAAA;AAC/B,IAAM,WAAW,GAAG,aAAa,CAAA;AACjC,IAAM,gBAAgB,GAAG,kBAAkB,CAAA;AAE3C,SAAgB,WAAW,CAAC,MAAoB;IAC9C,OAAO,mBAAmB,CAAC,MAAM,CAAC,CAAA;AACpC,CAAC;AAFD,kCAEC;AAED,SAAS,mBAAmB,CAAC,MAAoB;IAC/C,IAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAA;IAE9B,IAAM,SAAS,GAAmB,EAAE,CAAA;IACpC,SAAS,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAA;IAE5B,IAAI,CAAC,IAAA,qBAAW,EAAC,OAAO,CAAC,EAAE;QACzB,SAAS,CAAC,OAAO,GAAG,OAAO,CAAA;KAC5B;IAED,IAAI,IAAA,aAAG,EAAC,MAAM,EAAE,MAAM,CAAC,EAAE;QACvB,MAAM,CACJ,+CAA+C;YAC/C,8FAA8F,CAC/F,CAAA;KACF;IAED,IAAI,IAAA,aAAG,EAAC,MAAM,EAAE,UAAU,CAAC,EAAE;QAC3B,mEAAmE;QACnE,SAAS,CAAC,UAAU,GAAQ,MAAM,CAAC,UAAU,CAAC,CAAA;KAC/C;IAED,IAAA,0BAAiB,EAAC,CAAC,SAAS,CAAC,CAAC,CAAA;IAE9B,IAAI,IAAA,aAAG,EAAC,MAAM,EAAE,KAAK,CAAC,EAAE;QACtB,SAAS,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;KAChC;IAED,IAAI,IAAA,aAAG,EAAC,MAAM,EAAE,KAAK,CAAC,EAAE;QACtB,SAAS,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;KAChC;IAED,IAAI,IAAA,aAAG,EAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;QACzB,SAAS,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAA;KACtC;IAED,IAAI,IAAA,aAAG,EAAC,MAAM,EAAE,SAAS,CAAC,EAAE;QAC1B,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,CAAA;KACxC;IAED,IAAI,IAAA,aAAG,EAAC,MAAM,EAAE,UAAU,CAAC,EAAE;QAC3B,SAAS,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAA;KAC1C;IAED,IAAI,IAAA,aAAG,EAAC,MAAM,EAAE,WAAW,CAAC,EAAE;QAC5B,SAAS,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAA;KAC5C;IAED,IAAI,IAAA,aAAG,EAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE;QACjC,SAAS,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAA;KACtD;IAED,OAAO,SAAS,CAAA;AAClB,CAAC;AAEY,QAAA,GAAG,GAAG,WAAW,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,oBAAK,CAAC,EAAE,EAAE,CAAC,CAAA;AAClE,IAAA,0BAAiB,EAAC,CAAC,WAAG,CAAC,CAAC,CAAA;AAExB,SAAgB,mBAAmB,CACjC,OAAkB,EAClB,KAAa,EACb,WAAmB,EACnB,SAAiB,EACjB,SAAiB,EACjB,OAAe,EACf,WAAmB,EACnB,SAAiB;IAEjB,OAAO;QACL,KAAK,OAAA;QACL,WAAW,aAAA;QACX,SAAS,WAAA;QACT,SAAS,WAAA;QACT,OAAO,SAAA;QACP,WAAW,aAAA;QACX,SAAS,WAAA;QACT,YAAY,EAAQ,OAAQ,CAAC,YAAY;QACzC,SAAS,EAAE,OAAO;KACnB,CAAA;AACH,CAAC;AArBD,kDAqBC;AAED,SAAgB,YAAY,CAAC,KAAa,EAAE,OAAkB;IAC5D,OAAO,IAAA,+BAAsB,EAAC,KAAK,EAAE,OAAO,CAAC,CAAA;AAC/C,CAAC;AAFD,oCAEC"}