Premier commit déjà bien avancé

This commit is contained in:
2025-11-10 18:33:24 +01:00
commit db4f0508cb
652 changed files with 440521 additions and 0 deletions

View File

@ -0,0 +1,21 @@
MIT License
Copyright (C) 2024 by Marijn Haverbeke <marijn@haverbeke.berlin>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View File

@ -0,0 +1,28 @@
# @marijn/find-cluster-break
Small JavaScript module for finding grapheme cluster breaks in
strings, scanning from a given position.
```javascript
import {findClusterBreak} from "@marijn/find-cluster-break"
console.log(findClusterBreak("💪🏽🦋", 0))
// → 4
```
This code is open source, released under an MIT license.
## Documentation
**`findClusterBreak`**`(str: string, pos: number, forward = true, includeExtending = true): number`
Returns a next grapheme cluster break _after_ (not equal to) `pos`,
if `forward` is true, or before otherwise. Returns `pos` itself if no
further cluster break is available in the string. Moves across
surrogate pairs, extending characters (when `includeExtending` is
true, which is the default), characters joined with zero-width joiners,
and flag emoji.
**`isExtendingChar`**`(code: number): boolean`
Query whether the given character has a `Grapheme_Cluster_Break` value
of `Extend` in Unicode.

View File

@ -0,0 +1,85 @@
'use strict';
// These are filled with ranges (rangeFrom[i] up to but not including
// rangeTo[i]) of code points that count as extending characters.
let rangeFrom = [], rangeTo = []
;(() => {
// Compressed representation of the Grapheme_Cluster_Break=Extend
// information from
// http://www.unicode.org/Public/16.0.0/ucd/auxiliary/GraphemeBreakProperty.txt.
// Each pair of elements represents a range, as an offet from the
// previous range and a length. Numbers are in base-36, with the empty
// string being a shorthand for 1.
let numbers = "lc,34,7n,7,7b,19,,,,2,,2,,,20,b,1c,l,g,,2t,7,2,6,2,2,,4,z,,u,r,2j,b,1m,9,9,,o,4,,9,,3,,5,17,3,3b,f,,w,1j,,,,4,8,4,,3,7,a,2,t,,1m,,,,2,4,8,,9,,a,2,q,,2,2,1l,,4,2,4,2,2,3,3,,u,2,3,,b,2,1l,,4,5,,2,4,,k,2,m,6,,,1m,,,2,,4,8,,7,3,a,2,u,,1n,,,,c,,9,,14,,3,,1l,3,5,3,,4,7,2,b,2,t,,1m,,2,,2,,3,,5,2,7,2,b,2,s,2,1l,2,,,2,4,8,,9,,a,2,t,,20,,4,,2,3,,,8,,29,,2,7,c,8,2q,,2,9,b,6,22,2,r,,,,,,1j,e,,5,,2,5,b,,10,9,,2u,4,,6,,2,2,2,p,2,4,3,g,4,d,,2,2,6,,f,,jj,3,qa,3,t,3,t,2,u,2,1s,2,,7,8,,2,b,9,,19,3,3b,2,y,,3a,3,4,2,9,,6,3,63,2,2,,1m,,,7,,,,,2,8,6,a,2,,1c,h,1r,4,1c,7,,,5,,14,9,c,2,w,4,2,2,,3,1k,,,2,3,,,3,1m,8,2,2,48,3,,d,,7,4,,6,,3,2,5i,1m,,5,ek,,5f,x,2da,3,3x,,2o,w,fe,6,2x,2,n9w,4,,a,w,2,28,2,7k,,3,,4,,p,2,5,,47,2,q,i,d,,12,8,p,b,1a,3,1c,,2,4,2,2,13,,1v,6,2,2,2,2,c,,8,,1b,,1f,,,3,2,2,5,2,,,16,2,8,,6m,,2,,4,,fn4,,kh,g,g,g,a6,2,gt,,6a,,45,5,1ae,3,,2,5,4,14,3,4,,4l,2,fx,4,ar,2,49,b,4w,,1i,f,1k,3,1d,4,2,2,1x,3,10,5,,8,1q,,c,2,1g,9,a,4,2,,2n,3,2,,,2,6,,4g,,3,8,l,2,1l,2,,,,,m,,e,7,3,5,5f,8,2,3,,,n,,29,,2,6,,,2,,,2,,2,6j,,2,4,6,2,,2,r,2,2d,8,2,,,2,2y,,,,2,6,,,2t,3,2,4,,5,77,9,,2,6t,,a,2,,,4,,40,4,2,2,4,,w,a,14,6,2,4,8,,9,6,2,3,1a,d,,2,ba,7,,6,,,2a,m,2,7,,2,,2,3e,6,3,,,2,,7,,,20,2,3,,,,9n,2,f0b,5,1n,7,t4,,1r,4,29,,f5k,2,43q,,,3,4,5,8,8,2,7,u,4,44,3,1iz,1j,4,1e,8,,e,,m,5,,f,11s,7,,h,2,7,,2,,5,79,7,c5,4,15s,7,31,7,240,5,gx7k,2o,3k,6o".split(",").map(s => s ? parseInt(s, 36) : 1);
for (let i = 0, n = 0; i < numbers.length; i++)
(i % 2 ? rangeTo : rangeFrom).push(n = n + numbers[i]);
})();
function isExtendingChar(code) {
if (code < 768) return false
for (let from = 0, to = rangeFrom.length;;) {
let mid = (from + to) >> 1;
if (code < rangeFrom[mid]) to = mid;
else if (code >= rangeTo[mid]) from = mid + 1;
else return true
if (from == to) return false
}
}
function isRegionalIndicator(code) {
return code >= 0x1F1E6 && code <= 0x1F1FF
}
const ZWJ = 0x200d;
function findClusterBreak(str, pos, forward = true, includeExtending = true) {
return (forward ? nextClusterBreak : prevClusterBreak)(str, pos, includeExtending)
}
function nextClusterBreak(str, pos, includeExtending) {
if (pos == str.length) return pos
// If pos is in the middle of a surrogate pair, move to its start
if (pos && surrogateLow(str.charCodeAt(pos)) && surrogateHigh(str.charCodeAt(pos - 1))) pos--;
let prev = codePointAt(str, pos);
pos += codePointSize(prev);
while (pos < str.length) {
let next = codePointAt(str, pos);
if (prev == ZWJ || next == ZWJ || includeExtending && isExtendingChar(next)) {
pos += codePointSize(next);
prev = next;
} else if (isRegionalIndicator(next)) {
let countBefore = 0, i = pos - 2;
while (i >= 0 && isRegionalIndicator(codePointAt(str, i))) { countBefore++; i -= 2; }
if (countBefore % 2 == 0) break
else pos += 2;
} else {
break
}
}
return pos
}
function prevClusterBreak(str, pos, includeExtending) {
while (pos > 0) {
let found = nextClusterBreak(str, pos - 2, includeExtending);
if (found < pos) return found
pos--;
}
return 0
}
function codePointAt(str, pos) {
let code0 = str.charCodeAt(pos);
if (!surrogateHigh(code0) || pos + 1 == str.length) return code0
let code1 = str.charCodeAt(pos + 1);
if (!surrogateLow(code1)) return code0
return ((code0 - 0xd800) << 10) + (code1 - 0xdc00) + 0x10000
}
function surrogateLow(ch) { return ch >= 0xDC00 && ch < 0xE000 }
function surrogateHigh(ch) { return ch >= 0xD800 && ch < 0xDC00 }
function codePointSize(code) { return code < 0x10000 ? 1 : 2 }
exports.findClusterBreak = findClusterBreak;
exports.isExtendingChar = isExtendingChar;

View File

@ -0,0 +1,15 @@
/**
Query whether the given character has a `Grapheme_Cluster_Break`
value of `Extend` in Unicode.
*/
export declare function isExtendingChar(code: number): boolean
/**
Returns a next grapheme cluster break _after_ (not equal to) `pos`,
if `forward` is true, or before otherwise. Returns `pos` itself if no
further cluster break is available in the string. Moves across
surrogate pairs, extending characters (when `includeExtending` is
true, which is the default), characters joined with zero-width joiners,
and flag emoji.
*/
export declare function findClusterBreak(str: string, pos: number, forward?: boolean, includeExtending?: boolean): number

View File

@ -0,0 +1,35 @@
{
"name": "@marijn/find-cluster-break",
"version": "1.0.2",
"type": "module",
"description": "Find the position of grapheme cluster breaks in a string",
"main": "src/index.js",
"exports": {
"import": "./src/index.js",
"require": "./dist/index.cjs"
},
"scripts": {
"test": "mocha test/*.js",
"prepare": "rollup -c"
},
"repository": {
"type": "git",
"url": "git+https://github.com/marijnh/find-cluster-break.git"
},
"keywords": [
"unicode",
"grapheme",
"cluster",
"break"
],
"author": "Marijn Haverbeke <marijn@haverbeke.berlin>",
"license": "MIT",
"bugs": {
"url": "https://github.com/marijnh/find-cluster-break/issues"
},
"homepage": "https://github.com/marijnh/find-cluster-break#readme",
"devDependencies": {
"mocha": "^10.7.3",
"rollup": "^4.28.1"
}
}

View File

@ -0,0 +1,7 @@
export default {
input: "src/index.js",
output: {
file: "dist/index.cjs",
format: "cjs"
}
}

View File

@ -0,0 +1,15 @@
/**
Query whether the given character has a `Grapheme_Cluster_Break`
value of `Extend` in Unicode.
*/
export declare function isExtendingChar(code: number): boolean
/**
Returns a next grapheme cluster break _after_ (not equal to) `pos`,
if `forward` is true, or before otherwise. Returns `pos` itself if no
further cluster break is available in the string. Moves across
surrogate pairs, extending characters (when `includeExtending` is
true, which is the default), characters joined with zero-width joiners,
and flag emoji.
*/
export declare function findClusterBreak(str: string, pos: number, forward?: boolean, includeExtending?: boolean): number

View File

@ -0,0 +1,87 @@
// These are filled with ranges (rangeFrom[i] up to but not including
// rangeTo[i]) of code points that count as extending characters.
let rangeFrom = [], rangeTo = []
;(() => {
// Compressed representation of the Grapheme_Cluster_Break=Extend
// information from
// http://www.unicode.org/Public/16.0.0/ucd/auxiliary/GraphemeBreakProperty.txt.
// Each pair of elements represents a range, as an offet from the
// previous range and a length. Numbers are in base-36, with the empty
// string being a shorthand for 1.
let numbers = "lc,34,7n,7,7b,19,,,,2,,2,,,20,b,1c,l,g,,2t,7,2,6,2,2,,4,z,,u,r,2j,b,1m,9,9,,o,4,,9,,3,,5,17,3,3b,f,,w,1j,,,,4,8,4,,3,7,a,2,t,,1m,,,,2,4,8,,9,,a,2,q,,2,2,1l,,4,2,4,2,2,3,3,,u,2,3,,b,2,1l,,4,5,,2,4,,k,2,m,6,,,1m,,,2,,4,8,,7,3,a,2,u,,1n,,,,c,,9,,14,,3,,1l,3,5,3,,4,7,2,b,2,t,,1m,,2,,2,,3,,5,2,7,2,b,2,s,2,1l,2,,,2,4,8,,9,,a,2,t,,20,,4,,2,3,,,8,,29,,2,7,c,8,2q,,2,9,b,6,22,2,r,,,,,,1j,e,,5,,2,5,b,,10,9,,2u,4,,6,,2,2,2,p,2,4,3,g,4,d,,2,2,6,,f,,jj,3,qa,3,t,3,t,2,u,2,1s,2,,7,8,,2,b,9,,19,3,3b,2,y,,3a,3,4,2,9,,6,3,63,2,2,,1m,,,7,,,,,2,8,6,a,2,,1c,h,1r,4,1c,7,,,5,,14,9,c,2,w,4,2,2,,3,1k,,,2,3,,,3,1m,8,2,2,48,3,,d,,7,4,,6,,3,2,5i,1m,,5,ek,,5f,x,2da,3,3x,,2o,w,fe,6,2x,2,n9w,4,,a,w,2,28,2,7k,,3,,4,,p,2,5,,47,2,q,i,d,,12,8,p,b,1a,3,1c,,2,4,2,2,13,,1v,6,2,2,2,2,c,,8,,1b,,1f,,,3,2,2,5,2,,,16,2,8,,6m,,2,,4,,fn4,,kh,g,g,g,a6,2,gt,,6a,,45,5,1ae,3,,2,5,4,14,3,4,,4l,2,fx,4,ar,2,49,b,4w,,1i,f,1k,3,1d,4,2,2,1x,3,10,5,,8,1q,,c,2,1g,9,a,4,2,,2n,3,2,,,2,6,,4g,,3,8,l,2,1l,2,,,,,m,,e,7,3,5,5f,8,2,3,,,n,,29,,2,6,,,2,,,2,,2,6j,,2,4,6,2,,2,r,2,2d,8,2,,,2,2y,,,,2,6,,,2t,3,2,4,,5,77,9,,2,6t,,a,2,,,4,,40,4,2,2,4,,w,a,14,6,2,4,8,,9,6,2,3,1a,d,,2,ba,7,,6,,,2a,m,2,7,,2,,2,3e,6,3,,,2,,7,,,20,2,3,,,,9n,2,f0b,5,1n,7,t4,,1r,4,29,,f5k,2,43q,,,3,4,5,8,8,2,7,u,4,44,3,1iz,1j,4,1e,8,,e,,m,5,,f,11s,7,,h,2,7,,2,,5,79,7,c5,4,15s,7,31,7,240,5,gx7k,2o,3k,6o".split(",").map(s => s ? parseInt(s, 36) : 1)
for (let i = 0, n = 0; i < numbers.length; i++)
(i % 2 ? rangeTo : rangeFrom).push(n = n + numbers[i])
})()
export function isExtendingChar(code) {
if (code < 768) return false
for (let from = 0, to = rangeFrom.length;;) {
let mid = (from + to) >> 1
if (code < rangeFrom[mid]) to = mid
else if (code >= rangeTo[mid]) from = mid + 1
else return true
if (from == to) return false
}
}
function isRegionalIndicator(code) {
return code >= 0x1F1E6 && code <= 0x1F1FF
}
function check(code) {
for (let i = 0; i < rangeFrom.length; i++) {
if (rangeTo[i] > code) return rangeFrom[i] <= code
}
return false
}
const ZWJ = 0x200d
export function findClusterBreak(str, pos, forward = true, includeExtending = true) {
return (forward ? nextClusterBreak : prevClusterBreak)(str, pos, includeExtending)
}
function nextClusterBreak(str, pos, includeExtending) {
if (pos == str.length) return pos
// If pos is in the middle of a surrogate pair, move to its start
if (pos && surrogateLow(str.charCodeAt(pos)) && surrogateHigh(str.charCodeAt(pos - 1))) pos--
let prev = codePointAt(str, pos)
pos += codePointSize(prev)
while (pos < str.length) {
let next = codePointAt(str, pos)
if (prev == ZWJ || next == ZWJ || includeExtending && isExtendingChar(next)) {
pos += codePointSize(next)
prev = next
} else if (isRegionalIndicator(next)) {
let countBefore = 0, i = pos - 2
while (i >= 0 && isRegionalIndicator(codePointAt(str, i))) { countBefore++; i -= 2 }
if (countBefore % 2 == 0) break
else pos += 2
} else {
break
}
}
return pos
}
function prevClusterBreak(str, pos, includeExtending) {
while (pos > 0) {
let found = nextClusterBreak(str, pos - 2, includeExtending)
if (found < pos) return found
pos--
}
return 0
}
function codePointAt(str, pos) {
let code0 = str.charCodeAt(pos)
if (!surrogateHigh(code0) || pos + 1 == str.length) return code0
let code1 = str.charCodeAt(pos + 1)
if (!surrogateLow(code1)) return code0
return ((code0 - 0xd800) << 10) + (code1 - 0xdc00) + 0x10000
}
function surrogateLow(ch) { return ch >= 0xDC00 && ch < 0xE000 }
function surrogateHigh(ch) { return ch >= 0xD800 && ch < 0xDC00 }
function codePointSize(code) { return code < 0x10000 ? 1 : 2 }

View File

@ -0,0 +1,30 @@
import {findClusterBreak} from "../src/index.js"
function assertEq(a, b) {
if (a !== b) throw new Error(`${a} !== ${b}`)
}
describe("findClusterBreak", () => {
function test(spec) {
it(spec, () => {
let breaks = [], next
while ((next = spec.indexOf("|")) > -1) {
breaks.push(next)
spec = spec.slice(0, next) + spec.slice(next + 1)
}
let found = []
for (let i = 0;;) {
let next = findClusterBreak(spec, i)
if (next == spec.length) break
found.push(i = next)
}
assertEq(found.join(","), breaks.join(","))
})
}
test("a|b|c|d")
test("a|é̠|ő|x")
test("😎|🙉")
test("👨‍🎤|💪🏽|👩‍👩‍👧‍👦|❤")
test("🇩🇪|🇫🇷|🇪🇸|x|🇮🇹")
})