From 9097d93ac7f0e173089e3f7453ba021116811398 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Fri, 13 Oct 2023 17:07:47 +0100 Subject: [PATCH 1/4] Add shared library for filepath normalization --- .../utils/FilepathNormalizeTest.expected | 10 +++ .../utils/FilepathNormalizeTest.ql | 14 ++++ .../2023-10-13-filepath-normalization.md | 4 + shared/util/codeql/util/FilePath.qll | 74 +++++++++++++++++++ 4 files changed, 102 insertions(+) create mode 100644 csharp/ql/test/library-tests/utils/FilepathNormalizeTest.expected create mode 100644 csharp/ql/test/library-tests/utils/FilepathNormalizeTest.ql create mode 100644 shared/util/change-notes/2023-10-13-filepath-normalization.md create mode 100644 shared/util/codeql/util/FilePath.qll diff --git a/csharp/ql/test/library-tests/utils/FilepathNormalizeTest.expected b/csharp/ql/test/library-tests/utils/FilepathNormalizeTest.expected new file mode 100644 index 00000000000..0a43c01b186 --- /dev/null +++ b/csharp/ql/test/library-tests/utils/FilepathNormalizeTest.expected @@ -0,0 +1,10 @@ +| | . | +| ./a/b/c/../d | a/b/d | +| / | / | +| /a/b/../c | /a/c | +| /a/b/c/../../d/ | /a/d | +| a/.. | . | +| a/b/../c | a/c | +| a/b//////c/./d/../e//d// | a/b/c/e/d | +| a/b/c | a/b/c | +| a/b/c/../../../../d/e/../f | ../d/f | diff --git a/csharp/ql/test/library-tests/utils/FilepathNormalizeTest.ql b/csharp/ql/test/library-tests/utils/FilepathNormalizeTest.ql new file mode 100644 index 00000000000..4d9a2855dc9 --- /dev/null +++ b/csharp/ql/test/library-tests/utils/FilepathNormalizeTest.ql @@ -0,0 +1,14 @@ +import codeql.util.FilePath + +class FilepathTest extends NormalizableFilepath { + FilepathTest() { + this = + [ + "a/b/c", "a/b/../c", "a/..", "/a/b/../c", "a/b/c/../../../../d/e/../f", "", "/", + "./a/b/c/../d", "a/b//////c/./d/../e//d//", "/a/b/c/../../d/" + ] + } +} + +from FilepathTest s +select s, s.getNormalizedPath() diff --git a/shared/util/change-notes/2023-10-13-filepath-normalization.md b/shared/util/change-notes/2023-10-13-filepath-normalization.md new file mode 100644 index 00000000000..6324634fd58 --- /dev/null +++ b/shared/util/change-notes/2023-10-13-filepath-normalization.md @@ -0,0 +1,4 @@ +category: feature +--- +* Added `FilePath` API for normalizing filepaths. +``` \ No newline at end of file diff --git a/shared/util/codeql/util/FilePath.qll b/shared/util/codeql/util/FilePath.qll new file mode 100644 index 00000000000..b2906f5d4e0 --- /dev/null +++ b/shared/util/codeql/util/FilePath.qll @@ -0,0 +1,74 @@ +/** Provides a utility for normalizing filepaths. */ + +/** + * A filepath that should be normalized. + * Extend to provide additional strings that should be normalized as filepaths. + */ +abstract class NormalizableFilepath extends string { + bindingset[this] + NormalizableFilepath() { any() } + + private string getComponent(int i) { result = this.splitAt("/", i) } + + private int getNumComponents() { result = strictcount(int i | exists(this.getComponent(i))) } + + /** count .. segments in prefix in normalization from index i */ + private int dotdotCountFrom(int i) { + result = 0 and i = this.getNumComponents() + or + exists(string c | c = this.getComponent(i) | + if c = "" + then result = this.dotdotCountFrom(i + 1) + else + if c = "." + then result = this.dotdotCountFrom(i + 1) + else + if c = ".." + then result = this.dotdotCountFrom(i + 1) + 1 + else result = (this.dotdotCountFrom(i + 1) - 1).maximum(0) + ) + } + + /** count non-.. segments in postfix in normalization from index 0 to i-1 */ + private int segmentCountUntil(int i) { + result = 0 and i = 0 + or + exists(string c | c = this.getComponent(i - 1) | + if c = "" + then result = this.segmentCountUntil(i - 1) + else + if c = "." + then result = this.segmentCountUntil(i - 1) + else + if c = ".." + then result = (this.segmentCountUntil(i - 1) - 1).maximum(0) + else result = this.segmentCountUntil(i - 1) + 1 + ) + } + + private string part(int i) { + result = this.getComponent(i) and + result != "." and + if result = "" + then i = 0 + else ( + result != ".." and + 0 = this.dotdotCountFrom(i + 1) + or + result = ".." and + 0 = this.segmentCountUntil(i) + ) + } + + /** Gets the normalized filepath for this string; traversing `/../` paths. */ + string getNormalizedPath() { + exists(string norm | norm = concat(string s, int i | s = this.part(i) | s, "/" order by i) | + if norm != "" + then result = norm + else + if this.matches("/%") + then result = "/" + else result = "." + ) + } +} From aade79f723087f2d1e1a6dc95363b85049b69842 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Mon, 16 Oct 2023 13:02:28 +0100 Subject: [PATCH 2/4] Improve qldoc and fix changenote --- .../2023-10-13-filepath-normalization.md | 4 ++-- shared/util/codeql/util/FilePath.qll | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/shared/util/change-notes/2023-10-13-filepath-normalization.md b/shared/util/change-notes/2023-10-13-filepath-normalization.md index 6324634fd58..3ffdadc85b3 100644 --- a/shared/util/change-notes/2023-10-13-filepath-normalization.md +++ b/shared/util/change-notes/2023-10-13-filepath-normalization.md @@ -1,4 +1,4 @@ +--- category: feature --- -* Added `FilePath` API for normalizing filepaths. -``` \ No newline at end of file +* Added `FilePath` API for normalizing filepaths. \ No newline at end of file diff --git a/shared/util/codeql/util/FilePath.qll b/shared/util/codeql/util/FilePath.qll index b2906f5d4e0..c532f198d64 100644 --- a/shared/util/codeql/util/FilePath.qll +++ b/shared/util/codeql/util/FilePath.qll @@ -8,11 +8,13 @@ abstract class NormalizableFilepath extends string { bindingset[this] NormalizableFilepath() { any() } + /** Gets the `i`th path component of this string. */ private string getComponent(int i) { result = this.splitAt("/", i) } + /** Gets the number of path components of thi string. */ private int getNumComponents() { result = strictcount(int i | exists(this.getComponent(i))) } - /** count .. segments in prefix in normalization from index i */ + /** In the normalized path starting from component `i`, counts the number of `..` segments that path starts with. */ private int dotdotCountFrom(int i) { result = 0 and i = this.getNumComponents() or @@ -29,7 +31,7 @@ abstract class NormalizableFilepath extends string { ) } - /** count non-.. segments in postfix in normalization from index 0 to i-1 */ + /** In the normalized path up to (excluding) component `i`, counts the number of non-`..` segments that path ends with. */ private int segmentCountUntil(int i) { result = 0 and i = 0 or @@ -46,6 +48,7 @@ abstract class NormalizableFilepath extends string { ) } + /** Gets the `i`th component if that component should be included in the normalized path. */ private string part(int i) { result = this.getComponent(i) and result != "." and @@ -60,7 +63,13 @@ abstract class NormalizableFilepath extends string { ) } - /** Gets the normalized filepath for this string; traversing `/../` paths. */ + /** + * Gets the normalized filepath for this string. + * Normalizes `..` paths, `.` paths, and multiple `/`s as much as possible, but does not normalize case, resolve symlinks, or make relative paths absolute. + * Th normalized path will be absolute (begin with `/`) if and only if the original path is. + * The normalized path will not have a trailing `/`. + * Only `/` is treated as a path separator. + */ string getNormalizedPath() { exists(string norm | norm = concat(string s, int i | s = this.part(i) | s, "/" order by i) | if norm != "" From 05162c68ec6639d6fdb9ef23dc79000a9ac0a3d1 Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Mon, 16 Oct 2023 13:43:06 +0100 Subject: [PATCH 3/4] Fix typo --- shared/util/codeql/util/FilePath.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/util/codeql/util/FilePath.qll b/shared/util/codeql/util/FilePath.qll index c532f198d64..feb99face42 100644 --- a/shared/util/codeql/util/FilePath.qll +++ b/shared/util/codeql/util/FilePath.qll @@ -66,7 +66,7 @@ abstract class NormalizableFilepath extends string { /** * Gets the normalized filepath for this string. * Normalizes `..` paths, `.` paths, and multiple `/`s as much as possible, but does not normalize case, resolve symlinks, or make relative paths absolute. - * Th normalized path will be absolute (begin with `/`) if and only if the original path is. + * The normalized path will be absolute (begin with `/`) if and only if the original path is. * The normalized path will not have a trailing `/`. * Only `/` is treated as a path separator. */ From aa418dc7d016048e30cc7cced93d25ebbf245dbb Mon Sep 17 00:00:00 2001 From: Joe Farebrother Date: Tue, 17 Oct 2023 12:51:22 +0100 Subject: [PATCH 4/4] Add more line breaks in qldoc --- shared/util/codeql/util/FilePath.qll | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shared/util/codeql/util/FilePath.qll b/shared/util/codeql/util/FilePath.qll index feb99face42..1b047f3c91c 100644 --- a/shared/util/codeql/util/FilePath.qll +++ b/shared/util/codeql/util/FilePath.qll @@ -2,6 +2,7 @@ /** * A filepath that should be normalized. + * * Extend to provide additional strings that should be normalized as filepaths. */ abstract class NormalizableFilepath extends string { @@ -65,9 +66,13 @@ abstract class NormalizableFilepath extends string { /** * Gets the normalized filepath for this string. + * * Normalizes `..` paths, `.` paths, and multiple `/`s as much as possible, but does not normalize case, resolve symlinks, or make relative paths absolute. + * * The normalized path will be absolute (begin with `/`) if and only if the original path is. + * * The normalized path will not have a trailing `/`. + * * Only `/` is treated as a path separator. */ string getNormalizedPath() {