From 208112796848e37765eadae7ece16d1e957a8758 Mon Sep 17 00:00:00 2001 From: Parker Lougheed Date: Tue, 17 Mar 2026 00:44:45 +0800 Subject: [PATCH 1/4] Integrate packages from site-shared --- pkgs/analysis_defaults/analysis_options.yaml | 1 + pkgs/analysis_defaults/lib/analysis.yaml | 26 + pkgs/analysis_defaults/pubspec.yaml | 12 + pkgs/excerpter/LICENSE | 22 + pkgs/excerpter/README.md | 238 ++++++++ pkgs/excerpter/analysis_options.yaml | 5 + pkgs/excerpter/bin/excerpter.dart | 141 +++++ pkgs/excerpter/lib/excerpter.dart | 9 + pkgs/excerpter/lib/src/extract.dart | 215 +++++++ pkgs/excerpter/lib/src/inject.dart | 542 ++++++++++++++++++ pkgs/excerpter/lib/src/transform.dart | 320 +++++++++++ pkgs/excerpter/lib/src/update.dart | 190 ++++++ pkgs/excerpter/pubspec.yaml | 25 + pkgs/excerpter/test/cli_test.dart | 30 + pkgs/excerpter/test/transform_test.dart | 242 ++++++++ pkgs/excerpter/test/updater_test.dart | 86 +++ pkgs/excerpter/test_data/example/dartdoc.dart | 4 + pkgs/excerpter/test_data/example/plaster.dart | 44 ++ .../test_data/example/simple_main.dart | 3 + .../test_data/example/simple_region.dart | 5 + .../test_data/example/transforms.dart | 6 + pkgs/excerpter/test_data/expected/dartdoc.md | 12 + pkgs/excerpter/test_data/expected/plaster.md | 46 ++ pkgs/excerpter/test_data/expected/simple.md | 17 + .../test_data/expected/transforms.md | 11 + pkgs/excerpter/test_data/src/dartdoc.md | 8 + pkgs/excerpter/test_data/src/plaster.md | 29 + pkgs/excerpter/test_data/src/simple.md | 13 + pkgs/excerpter/test_data/src/transforms.md | 7 + pubspec.yaml | 2 + site/pubspec.yaml | 5 +- .../lib/src/commands/analyze_dart.dart | 7 +- .../lib/src/commands/format_dart.dart | 13 +- tool/dash_site/pubspec.yaml | 10 +- 34 files changed, 2319 insertions(+), 27 deletions(-) create mode 100644 pkgs/analysis_defaults/analysis_options.yaml create mode 100644 pkgs/analysis_defaults/lib/analysis.yaml create mode 100644 pkgs/analysis_defaults/pubspec.yaml create mode 100644 pkgs/excerpter/LICENSE create mode 100644 pkgs/excerpter/README.md create mode 100644 pkgs/excerpter/analysis_options.yaml create mode 100644 pkgs/excerpter/bin/excerpter.dart create mode 100644 pkgs/excerpter/lib/excerpter.dart create mode 100644 pkgs/excerpter/lib/src/extract.dart create mode 100644 pkgs/excerpter/lib/src/inject.dart create mode 100644 pkgs/excerpter/lib/src/transform.dart create mode 100644 pkgs/excerpter/lib/src/update.dart create mode 100644 pkgs/excerpter/pubspec.yaml create mode 100644 pkgs/excerpter/test/cli_test.dart create mode 100644 pkgs/excerpter/test/transform_test.dart create mode 100644 pkgs/excerpter/test/updater_test.dart create mode 100644 pkgs/excerpter/test_data/example/dartdoc.dart create mode 100644 pkgs/excerpter/test_data/example/plaster.dart create mode 100644 pkgs/excerpter/test_data/example/simple_main.dart create mode 100644 pkgs/excerpter/test_data/example/simple_region.dart create mode 100644 pkgs/excerpter/test_data/example/transforms.dart create mode 100644 pkgs/excerpter/test_data/expected/dartdoc.md create mode 100644 pkgs/excerpter/test_data/expected/plaster.md create mode 100644 pkgs/excerpter/test_data/expected/simple.md create mode 100644 pkgs/excerpter/test_data/expected/transforms.md create mode 100644 pkgs/excerpter/test_data/src/dartdoc.md create mode 100644 pkgs/excerpter/test_data/src/plaster.md create mode 100644 pkgs/excerpter/test_data/src/simple.md create mode 100644 pkgs/excerpter/test_data/src/transforms.md diff --git a/pkgs/analysis_defaults/analysis_options.yaml b/pkgs/analysis_defaults/analysis_options.yaml new file mode 100644 index 00000000000..b9bdf805acf --- /dev/null +++ b/pkgs/analysis_defaults/analysis_options.yaml @@ -0,0 +1 @@ +include: package:analysis_defaults/analysis.yaml diff --git a/pkgs/analysis_defaults/lib/analysis.yaml b/pkgs/analysis_defaults/lib/analysis.yaml new file mode 100644 index 00000000000..1124c9938b6 --- /dev/null +++ b/pkgs/analysis_defaults/lib/analysis.yaml @@ -0,0 +1,26 @@ +include: package:dart_flutter_team_lints/analysis_options.yaml + +analyzer: + language: + strict-casts: true + strict-inference: true + strict-raw-types: true + +linter: + rules: + - deprecated_member_use_from_same_package + - discarded_futures + - eol_at_end_of_file + - implicit_reopen + - invalid_case_patterns + - matching_super_parameters + - missing_code_block_language_in_doc_comment + - no_literal_bool_comparisons + - no_self_assignments + - prefer_final_fields + - prefer_final_in_for_each + - prefer_final_locals + - unnecessary_breaks + - unnecessary_null_aware_operator_on_extension_on_nullable + - use_enums + - use_truncating_division diff --git a/pkgs/analysis_defaults/pubspec.yaml b/pkgs/analysis_defaults/pubspec.yaml new file mode 100644 index 00000000000..bbf8f5f2e21 --- /dev/null +++ b/pkgs/analysis_defaults/pubspec.yaml @@ -0,0 +1,12 @@ +name: analysis_defaults +description: Analysis defaults for Dart/Flutter site tools. +publish_to: none + +resolution: workspace +environment: + sdk: ^3.11.0 + +# NOTE: Code isn't allowed in this package. +# Don't add dependencies besides the underlying lints package. +dependencies: + dart_flutter_team_lints: ^3.5.2 diff --git a/pkgs/excerpter/LICENSE b/pkgs/excerpter/LICENSE new file mode 100644 index 00000000000..c1503e154c5 --- /dev/null +++ b/pkgs/excerpter/LICENSE @@ -0,0 +1,22 @@ +The MIT License + +Copyright (c) 2017 Dart Project Authors +Copyright (c) 2026 The Flutter Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pkgs/excerpter/README.md b/pkgs/excerpter/README.md new file mode 100644 index 00000000000..47131da6f66 --- /dev/null +++ b/pkgs/excerpter/README.md @@ -0,0 +1,238 @@ +# Markdown code-block excerpt updater + +Tooling to update code excerpts in Markdown documentation +from regions declared in source files elsewhere. + +> [!WARNING] +> This package is still a work-in-progress and not completely functional. + +## Install + +This package is not currently published on pub, +so you must use it as a [git][git-dep] or [path][path-dep] dependency. + +[git-dep]: https://dart.dev/tools/pub/dependencies#git-packages +[path-dep]: https://dart.dev/tools/pub/dependencies#path-packages + +## Defining excerpt regions + +The package pulls content from Dart files that are +optionally split up with code regions. + +To include lines within a code region, +add a Dart line comment before the first line in the region with +`#docregion ` where `` is your desired name for the region. +Then after the final line of the region, +add a Dart line comment with `#enddocregion ` with the same name. +You can open and close a docregion multiple times within a file. + +Region names must be a non-empty sequence of alphanumeric characters, +optionally containing dashes (`-`). + +A single docregion comment can open or close multiple regions +with a comma-delimited list. + +### Annotated example + +The following Dart snippet is an example of using +docregion comments in a few different ways. + +```dart +// #docregion imports +import 'dart:async'; +// #enddocregion imports + +// #docregion main, main-stub +void main() async { + // #enddocregion main-stub + print('Compute π using the Monte Carlo method.'); + await for (final estimate in computePi().take(500)) { + print('π ≅ $estimate'); + } + // #docregion main-stub +} +// #enddocregion main, main-stub + +/// Generates a stream of increasingly accurate estimates of π. +Stream computePi({int batch = 100000}) async* { + // ... +} +``` + +The regions defined in this Dart snippet are: +`imports`, `main`, `main-stub`, and the unnamed region for the entire file. + +Some of the regions defined in the example above include: + +- `imports` region: + + ```dart + import 'dart:async'; + ``` + +- `main-stub` region: + + ```dart + void main() async { + // ··· + } + ``` + +The `main-stub` region is discontinuous as it has a break in it. +When this package is run to update excerpts, each break is +replaced by a language-specific comment filled with a plaster marker (`...`). + +For details concerning the processing of plasters, see the +[code_excerpt_updater][] documentation. + +## Injecting excerpts + +To inject content from docregions or entire files into Markdown files, +you use a special syntax to inject into a code block +or configure the injecting logic. + +In both cases, the syntax starts with `\`. + +### Inject instruction + +Use an inject instruction to inject a docregion from a source file +into the current Markdown file, with optional arguments +to configure the injection. + +Inject instructions must precede a Markdown code block +that is denoted with the language in the target file. + +````md + +```dart +void main() {} +``` +```` + +The first unnamed argument, surrounded in double quotes, +is the path to source file to pull regions from, +relative to the base source path set by the CLI. +A specific region from the file can be specified in parentheses, +otherwise the entire file is extracted. + +The following table outlines the parameters supported by injection instructions. +Note that specified arguments, such as transformations, are applied +in the order they appear in the instruction. + +| Parameter | Argument | Description | +|-------------|------------------------|-------------------------------------------------------------------------------| +| `indent-by` | `int` | The amount to indent each line by. | +| `plaster` | `String` | The plaster template to use, or `none` to disable. | +| `skip` | `int` | The amount of lines to skip at the beginning if positive, or end if negative. | +| `take` | `int` | The amount of lines to take at the beginning if positive, or end if negative. | +| `remove` | `String\|RegExp` | Remove the lines containing the specified pattern. | +| `retain` | `String\|RegExp` | Keep the lines containing the specified pattern. | +| `from` | `String\|RegExp` | Keep the lines after and including the first one with the specified pattern. | +| `to` | `String\|RegExp` | Keep the lines before and including the first one with the specified pattern. | +| `replace` | [Replacement syntax][] | Replace text with the specified pattern to the specified string. | + +[Replacement syntax]: #replacement-syntax + +For parameters that accept a `RegExp`, +they follow the Dart VM's supported syntax, +and must be wrapped in forward slashes, such as `//`. +If you're passing a normal string, the forward slashes are unnecessary. + +### Replacement syntax + +The `replace` argument accepts one or more semicolon separated +regular expression and replacement expression pairs. +The replacement expressions can be simple strings or +include backreferences to numbered capture groups from the +regular expression using, `$&`, `$1`, `$2`, and so on. + +The following replace expression replaces text like `Hello world` +with `[!Hello!] world`: + +```md +replace="/(Hello)( world)/[!$1!]$2/g;" +``` + +Compared to other transforms, replace transforms are +applied on the entire excerpt rather than per line. + +### Set instruction + +Use a set instruction to configure one of the following for +subsequent injection instructions: + +- The base directory that source files for docregions are found in. +- The template to be used for plaster lines. +- Replace expressions that will run for every inject instruction. + +Only one set instruction argument can be used at a time, +and only one of each can exist in the file. +Subsequent set instructions of the same type override the previous ones. + +#### Modify the source file base path + +To set the base directory that source files for docregions are found in +to a subdirectory of the CLI provided one, use the `path-base` argument: + +```md + +``` + +#### Modify the plaster template + +To change the template used for plaster lines, use the `plaster` argument: + +```md + +``` + +If you want to use the default plaster content specified by the CLI, +you can use `$defaultPlaster` within the template: + +```md + +``` + +#### Add a global transform + +To add a transform expression that is applied to all subsequent excerpts +use the `replace` argument and the same [replacement syntax][] as above. + +```md + +``` + +[replacement syntax]: #replacement-syntax + +#### Reset set instructions + +To reset any of the instructions, +use the same arguments set to an empty string (`""`); + +## Updating excerpts + +To update the excerpts specified by injection instructions +within your Markdown files, you can either use the package +as a library through the `Updater` class, or the CLI. + +```bash +dart run excerpter [OPTIONS] +``` + +| Option | Description | +|---------------------|---------------------------------------------------------------------------------------------------------------------------------| +| `--dry-run ` | If the updater should only report if excerpts need to be updated. | +| `--fail-on-update` | Report a non-zero exit code if an excerpt is or needs to be updated. | +| `--exclude` | Regular expressions of paths to exclude when processing a directory recursively. Dot files and directories are always excluded. | +| `--base-source` | The path to the directory containing the source files that excerpt regions should be retrieved from. | +| `--plaster-content` | The default plaster content, such as "..." or "···". | +| `--replace` | A replacement to run on every excerpt. Refer to the [replacement syntax](#replacement-syntax) for more details. | + +## Learn more + +To learn more about the tool, check out +the various usages across the [dart.dev][] and [docs.flutter.dev][] +website repositories. + +[dart.dev]: https://github.com/dart-lang/site-www +[docs.flutter.dev]: https://github.com/flutter/website diff --git a/pkgs/excerpter/analysis_options.yaml b/pkgs/excerpter/analysis_options.yaml new file mode 100644 index 00000000000..b29fbd0aee8 --- /dev/null +++ b/pkgs/excerpter/analysis_options.yaml @@ -0,0 +1,5 @@ +include: package:analysis_defaults/analysis.yaml + +analyzer: + exclude: + - test_data/ diff --git a/pkgs/excerpter/bin/excerpter.dart b/pkgs/excerpter/bin/excerpter.dart new file mode 100644 index 00000000000..d64018a837b --- /dev/null +++ b/pkgs/excerpter/bin/excerpter.dart @@ -0,0 +1,141 @@ +// Copyright (c) 2023. All rights reserved. Use of this source code +// is governed by a MIT-style license that can be found in the LICENSE file. + +import 'dart:io' as io; + +import 'package:args/args.dart'; +import 'package:excerpter/excerpter.dart'; +import 'package:path/path.dart' as path; + +void main(final List args) async { + await runExcerpter(args); +} + +/// Run the code excerpter with the specified [arguments], +/// usually meant to be called from a command-line app. +Future runExcerpter(final List arguments) async { + final ArgResults results; + try { + results = _argParser.parse(arguments); + } on FormatException catch (e) { + _printUsageAndExit(message: e.message); + } + + final dryRun = results[_dryRunFlag] as bool? ?? false; + final failOnUpdate = results[_failOnUpdateFlag] as bool? ?? false; + final excludePaths = results[_excludeOption] as List? ?? const []; + final plasterContent = results[_plasterContentOption] as String? ?? '...'; + final replaceInstructions = results[_replaceOption] as String?; + final baseSourcePath = path.absolute( + results[_baseSourcePathOption] as String? ?? path.current, + ); + + if (results.rest.length != 1) { + _printUsageAndExit( + message: 'You must specify a file or directory to run updates on.', + ); + } + final updatePath = path.absolute(results.rest.first); + + final replaceTransforms = replaceInstructions == null + ? const [] + : stringToReplaceTransforms( + replaceInstructions, + (e) => _printUsageAndExit(message: e), + ); + + final updater = Updater( + baseSourcePath: baseSourcePath, + validTargetExtensions: const {'.md'}, + defaultPlasterContent: plasterContent, + defaultTransforms: replaceTransforms, + excludePaths: excludePaths.map(RegExp.new), + ); + + final result = await updater.update(updatePath, makeUpdates: !dryRun); + final warnings = result.warnings; + final warningCount = warnings.length; + final errors = result.errors; + final errorCount = errors.length; + if (warningCount > 0 || errorCount > 0) { + for (final error in errors) { + print(' error - $error'); + } + for (final warning in warnings) { + print('warning - $warning'); + } + print(''); + print('$errorCount errors and $warningCount warnings found!'); + print(''); + } + + print( + 'Processed ${result.filesVisited} out of ' + '${result.totalFilesToVisit} files: ' + '${result.excerptsNeedingUpdates} out of ' + '${result.excerptsVisited} excerpts visited ' + '${result.madeUpdates ? 'were updated' : 'need to be updated'}.', + ); + + if (result.errors.length case final amountOfErrors when amountOfErrors > 0) { + io.exitCode = amountOfErrors; + } else if (failOnUpdate && result.excerptsNeedingUpdates > 0) { + io.exitCode = result.excerptsNeedingUpdates; + } +} + +final _argParser = ArgParser() + ..addFlag( + _dryRunFlag, + negatable: false, + help: 'If the updater should only report if excerpts need to be updated.', + ) + ..addFlag( + _failOnUpdateFlag, + negatable: false, + help: + 'Report a non-zero exit code if ' + 'an excerpt is or needs to be updated.', + ) + ..addMultiOption( + _excludeOption, + help: + 'Regular expressions of paths to exclude when ' + 'processing a directory recursively.\n' + 'Dot files and directories are always excluded.', + ) + ..addOption( + _baseSourcePathOption, + help: + 'The path to the directory containing the source files that ' + 'excerpt regions should be retrieved from.', + ) + ..addOption( + _plasterContentOption, + help: 'The default plaster content, such as "..." or "···".', + ) + ..addOption( + _replaceOption, + help: + 'A replacement to run on every excerpt.\n' + 'Refer to the package docs for syntax help.', + ); + +/// Print the usage information for this command, +/// optionally with the specified error [message] and [exitCode], +/// then exit. +/// +/// If no [exitCode] is specified, exit with a code of `1`, indicating failure. +Never _printUsageAndExit({String? message, int exitCode = 1}) { + if (message != null) print('\n$message\n'); + print('Usage: excerpter [OPTIONS] file_or_directory\n'); + print(_argParser.usage); + io.exit(exitCode); +} + +const String _dryRunFlag = 'dry-run'; +const String _failOnUpdateFlag = 'fail-on-update'; +const String _excludeOption = 'exclude'; +const String _plasterContentOption = 'plaster-content'; +const String _replaceOption = 'replace'; +const String _baseSourcePathOption = 'base-source'; diff --git a/pkgs/excerpter/lib/excerpter.dart b/pkgs/excerpter/lib/excerpter.dart new file mode 100644 index 00000000000..48fce4eb16c --- /dev/null +++ b/pkgs/excerpter/lib/excerpter.dart @@ -0,0 +1,9 @@ +// Copyright (c) 2023. All rights reserved. Use of this source code +// is governed by a MIT-style license that can be found in the LICENSE file. + +/// Tooling to update code excerpts in Markdown documentation +/// from regions declared in source files elsewhere. +library; + +export 'src/transform.dart'; +export 'src/update.dart'; diff --git a/pkgs/excerpter/lib/src/extract.dart b/pkgs/excerpter/lib/src/extract.dart new file mode 100644 index 00000000000..1577a0133ed --- /dev/null +++ b/pkgs/excerpter/lib/src/extract.dart @@ -0,0 +1,215 @@ +// Copyright (c) 2023. All rights reserved. Use of this source code +// is governed by a MIT-style license that can be found in the LICENSE file. + +import 'dart:io'; +import 'dart:math' as math show min; + +import 'package:meta/meta.dart'; + +/// A tool to extract declared docregions from specified source files. +/// +/// Caches all docregions parsed from files, so consider +/// creating a new one for each file you're injecting in to. +final class ExcerptExtractor { + /// A cache of file paths to to the regions found within them. + /// + /// If the value a pile path points to is `null`, + /// the file couldn't be found or read. + final Map?> _regionCacheByPath = {}; + + /// Extract the region with the specified [regionName] from + /// the file located at the specified [path]. + /// + /// If a file does not exist at that location or the [regionName] + /// does not exist either, a `ExtractException` will be thrown. + @useResult + Future extractRegion(String path, String regionName) async { + if (!_regionCacheByPath.containsKey(path)) { + _regionCacheByPath[path] = await _extractAllRegions(path); + } + final regions = _regionCacheByPath[path]; + if (regions == null) { + throw ExtractException('No file exists at $path.'); + } + + final region = regions[regionName]; + if (region == null) { + throw ExtractException( + 'The region "$regionName" does not exist in the file at $path.', + ); + } + + return region; + } + + @useResult + Future?> _extractAllRegions(String path) async { + final file = File(path); + if (!(await file.exists())) { + return null; + } + + final lines = await file.readAsLines(); + if (lines.isEmpty) { + return const {}; + } + + final regionContent = {_entireFileRegionName: Region._()}; + final currentRegions = {_entireFileRegionName}; + + for (var lineIndex = 0; lineIndex < lines.length; lineIndex += 1) { + final line = lines[lineIndex]; + final trimmedLine = line.trimLeft(); + final indent = line.length - trimmedLine.length; + + final directive = _docRegionDirective.firstMatch(trimmedLine); + if (directive != null) { + final isEnd = directive.namedGroup('end') != null; + final rawRegionNames = directive.namedGroup('regions'); + if (rawRegionNames == null) { + throw const ExtractException( + 'A docregion comment must specify at least one region!', + ); + } + final regionNames = rawRegionNames.split(','); + for (final rawRegionName in regionNames) { + final regionName = rawRegionName.trim(); + if (regionName.isEmpty) { + throw const ExtractException( + 'docregion comment tried to use an empty region name.', + ); + } + if (isEnd) { + final removed = currentRegions.remove(regionName); + if (!removed) { + throw ExtractException( + 'enddocregion tried to close the ' + "unopened '$regionName' region!", + ); + } + } else { + if (regionContent[regionName] case final region?) { + // If the region already exists, add a plaster line. + region._addPlaster(indent); + } else { + regionContent[regionName] = Region._(); + } + + currentRegions.add(regionName); + } + } + } else { + // Just a normal line. + for (final region in currentRegions) { + regionContent[region]!._addLine(line, indent); + } + } + } + + currentRegions.remove(_entireFileRegionName); + if (currentRegions.isNotEmpty) { + throw ExtractException('Regions $currentRegions were not closed.'); + } + + return regionContent; + } +} + +/// The contents of a docregion found in a file. +final class Region { + /// The untransformed text lines or plaster lines of the docregion. + final List<_RegionLine> _lines = []; + + /// The minimum indent seen in this region. + /// + /// `99999` is the initial value as no line should be longer than that... + int _minIndent = 99999; + + /// Creates a [Region] with the specified indentation, + /// usually from the docregion comment. + Region._(); + + /// Adds the specified [line] with the specified [indent] + /// to the contents of the region. + void _addLine(String line, int indent) { + _lines.add(_StringLine(line)); + + // Ignore the indent of blank lines. + if (line.trim().isNotEmpty) { + _minIndent = math.min(_minIndent, indent); + } + } + + /// Adds a line where a plaster could be inserted + /// as well as the [directiveIndent] of the directive adding it. + /// + /// This is usually when a docregion is closed and opened again. + void _addPlaster(int directiveIndent) { + _lines.add(_PlasterLine(directiveIndent)); + } + + /// Builds a list of strings from the region, + /// replacing lines marked as plasters with the + /// specified [plaster] content, + /// and applying the minimum indentation to each line. + /// + /// If [plaster] is `null` or `'none'`, + /// the plaster lines are not included at all. + @useResult + Iterable linesWithPlaster(final String? plaster) { + final updatedLines = []; + final includePlaster = plaster != null && plaster != 'none'; + + for (final line in _lines) { + switch (line) { + case _PlasterLine(:final directiveIndent): + if (includePlaster) { + final minimizedDirectiveIndent = directiveIndent - _minIndent; + updatedLines.add('${' ' * minimizedDirectiveIndent}$plaster'); + } + case _StringLine(:final line): + if (_minIndent == 0 || line.length < _minIndent) { + updatedLines.add(line); + } else { + updatedLines.add(line.substring(_minIndent)); + } + } + } + + return updatedLines; + } +} + +sealed class _RegionLine {} + +final class _StringLine extends _RegionLine { + final String line; + + _StringLine(this.line); +} + +final class _PlasterLine extends _RegionLine { + final int directiveIndent; + + _PlasterLine(this.directiveIndent); +} + +/// An exception thrown when a [ExcerptExtractor] +/// failed to extract a region. +@immutable +final class ExtractException implements Exception { + /// The error causing the exception during extraction. + final String error; + + /// Create a [ExtractException] with the specified [error]. + const ExtractException(this.error); + + @override + String toString() => error; +} + +const String _entireFileRegionName = ''; + +final RegExp _docRegionDirective = RegExp( + r'^.*?#(?end)?docregion\s(?[a-zA-Z0-9,_\-\s]+).*?$', +); diff --git a/pkgs/excerpter/lib/src/inject.dart b/pkgs/excerpter/lib/src/inject.dart new file mode 100644 index 00000000000..d3d643c4c27 --- /dev/null +++ b/pkgs/excerpter/lib/src/inject.dart @@ -0,0 +1,542 @@ +// Copyright (c) 2023. All rights reserved. Use of this source code +// is governed by a MIT-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:collection/collection.dart'; +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as path; + +import 'extract.dart'; +import 'transform.dart'; + +/// An excerpt updater for an individual file that is +/// guaranteed to only run once. +final class FileUpdater { + /// The default transforms to run initially on all code excerpt regions. + final Iterable defaultTransforms; + + /// The path to the directory of the source files to pull regions from. + final String baseSourcePath; + + /// The content to include in plaster replacements by default. + final String defaultPlasterContent; + + /// The path to the file to update the excerpt regions of. + final String pathToUpdate; + + /// The update results for this file after + /// running [process] for the first time. + FileProcessResults? _results; + + /// Create a new [FileUpdater] that will process the file at the + /// specified [pathToUpdate], pull code excerpt regions from the + /// directory at [baseSourcePath], then optionally update + /// the file with updated regions. + /// + /// The [defaultPlasterContent] is used when applying plasters + /// if no the page or inject instruction doesn't override it. + /// + /// The [defaultTransforms] specified are ran to transform + /// each regions pulled from the source files. + FileUpdater( + this.pathToUpdate, { + required this.baseSourcePath, + required this.defaultPlasterContent, + required this.defaultTransforms, + }); + + /// Process the file at [pathToUpdate] and determine + /// what, if any, updates need to be made to its injected + /// code excerpts. + Future process() async { + if (_results case final results?) return results; + + final updatedContent = StringBuffer(); + final warnings = []; + + var excerptsVisited = 0; + final excerptsUpdated = <({int instructionLine, String updated})>[]; + + final extractor = ExcerptExtractor(); + + final originalLines = await File(pathToUpdate).readAsLines(); + + Iterable wholeFileTransforms = []; + String? wholeFilePlasterTemplate; + var wholeFilePathBase = ''; + + for (var lineIndex = 0; lineIndex < originalLines.length; lineIndex += 1) { + final line = originalLines[lineIndex]; + final trimmedLine = line.trimLeft(); + + // The line won't change whether if + // it's an instruction or not in a excerpt. + updatedContent.writeln(line); + if (!trimmedLine.startsWith(_instructionStart)) { + continue; + } + + final instructionLineNumber = lineIndex + 1; + final instructionIndent = line.length - trimmedLine.length; + + Never reportError(String error) { + throw InjectionException._( + pathToUpdate, + line, + instructionLineNumber, + error, + ); + } + + final instruction = _Instruction.fromLine(line, reportError); + if (instruction case _SetInstruction()) { + switch (instruction) { + case _SetPathBaseInstruction(:final pathBase): + wholeFilePathBase = pathBase; + case _SetPlasterInstruction(:final plasterTemplate): + wholeFilePlasterTemplate = plasterTemplate; + case _SetFileReplaceInstruction(:final transforms): + wholeFileTransforms = transforms; + } + } else if (instruction case _InjectInstruction()) { + // Move to index of what should be code block opening + lineIndex += 1; + + if (lineIndex >= originalLines.length) { + reportError( + 'An inject instruction must be followed by a code block. ' + 'Found end of file.', + ); + } + + final lineAfterInstruction = originalLines[lineIndex]; + final fencedCodeBlock = _codeBlockStart.firstMatch( + lineAfterInstruction, + ); + if (fencedCodeBlock == null) { + reportError( + 'An inject instruction must be followed by a code block ' + 'with a language specified.', + ); + } + + excerptsVisited += 1; + + final backticks = fencedCodeBlock.namedGroup('backticks')!; + final backtickCount = backticks.length; + final language = fencedCodeBlock.namedGroup('language')!; + final codeBlockEndMarker = RegExp('^\\s*`{$backtickCount}.*?\$'); + + final oldLines = []; + String? codeBlockClose; + + while (lineIndex < originalLines.length) { + lineIndex += 1; + final codeLine = originalLines[lineIndex]; + + if (codeBlockEndMarker.firstMatch(codeLine) != null) { + codeBlockClose = codeLine; + break; + } + + oldLines.add(codeLine); + } + + if (codeBlockClose == null) { + reportError('Unclosed or unmatched code block.'); + } + + final combinedPath = path.join( + baseSourcePath, + wholeFilePathBase, + instruction.targetPath, + ); + + final Region region; + try { + region = await extractor.extractRegion( + combinedPath, + instruction.regionName, + ); + } on ExtractException catch (e) { + reportError(e.error); + } + + var plaster = (instruction.plasterTemplate ?? wholeFilePlasterTemplate) + ?.replaceAll(r'$defaultPlaster', defaultPlasterContent); + + if (plaster == null) { + final languageComments = _contentTypeToCommentFormat(language); + if (languageComments case (:final prefix, :final suffix)) { + plaster = '$prefix$defaultPlasterContent$suffix'; + } + } + + var updatedLines = region.linesWithPlaster(plaster); + + final transforms = [ + ...instruction.transforms, + ...wholeFileTransforms, + ...defaultTransforms, + ]; + + for (final transform in transforms) { + updatedLines = transform.transform(updatedLines); + } + + updatedLines = updatedLines.map((line) => line.trimRight()); + + // Remove all shared whitespace on the left. + int? sharedLeftWhitespace; + for (final line in updatedLines) { + final leftWhitespace = line.length - line.trimLeft().length; + // If this line has less left whitespace than preceding lines, + // use its count as the shared left whitespace. + if (sharedLeftWhitespace == null || + leftWhitespace < sharedLeftWhitespace) { + sharedLeftWhitespace = leftWhitespace; + } + } + + if (sharedLeftWhitespace != null && sharedLeftWhitespace > 0) { + updatedLines = [ + for (final line in updatedLines) + line.substring(sharedLeftWhitespace), + ]; + } + + // Add back the indentation from the file and any from the instruction. + updatedLines = IndentTransform( + instructionIndent + (instruction.indentBy ?? 0), + ).transform(updatedLines); + + final updatedExcerpt = updatedLines.join('\n'); + if (!(const IterableEquality().equals( + oldLines, + updatedLines, + ))) { + excerptsUpdated.add(( + instructionLine: instructionLineNumber, + updated: updatedExcerpt, + )); + } + + updatedContent.writeln(lineAfterInstruction); + updatedContent.writeln(updatedExcerpt); + updatedContent.writeln(codeBlockClose); + } + } + + return FileProcessResults._( + pathToUpdate, + updatedContent.toString(), + excerptsVisited, + excerptsUpdated, + warnings, + ); + } +} + +/// The results of processing a file and its injection instructions. +@immutable +final class FileProcessResults { + /// The path of the file to process and update. + final String pathToUpdate; + + /// The content of the file after applying updates. + final String updatedContent; + + /// The amount of excerpts visited in the file. + final int excerptsVisited; + + /// The warnings experienced while processing the file. + final List warnings; + + /// All instances where an excerpt needs to be updated, + /// including the line number in the original file of the instruction + /// and the contents after updating. + final List<({int instructionLine, String updated})> excerptsUpdated; + + /// Create a [FileProcessResults] to communicate the results of + /// processing a file and its code excerpt instructions. + FileProcessResults._( + this.pathToUpdate, + this.updatedContent, + this.excerptsVisited, + this.excerptsUpdated, + this.warnings, + ); + + /// If any excerpts at [pathToUpdate] need updates. + bool get needsUpdates => excerptsUpdated.isNotEmpty; + + /// Write the excerpt updates determined as necessary + /// to the file at [pathToUpdate]. + Future writeUpdates() async { + await File(pathToUpdate).writeAsString(updatedContent); + } +} + +/// An exception that occurs when a code-excerpt injection fails. +@immutable +final class InjectionException implements Exception { + /// The path of the file this injection instruction exception occurred for. + final String filePath; + + /// The line of the injection instruction that had errors or caused errors. + final String line; + + /// The line number of the injection instruction that + /// had errors or caused errors. + /// + /// This is the line number before the updates, if any, were made. + final int lineNumber; + + /// The error that occurred due to the injection instruction. + final String error; + + /// Create an exception to convey that an injection instruction of [line] + /// at [lineNumber] in the file at [filePath] had an error + /// or caused an error. + InjectionException._(this.filePath, this.line, this.lineNumber, this.error); + + @override + String toString() => '$filePath:$lineNumber - $error'; +} + +final RegExp _instructionPattern = RegExp( + r'^\s*<\?code-excerpt\s+(?:"(?\S+)(?:\s\((?[^)]+)\))?\s*")?(?.*?)\?>$', +); + +final RegExp _instructionStart = RegExp(r'^<\?code-excerpt'); + +final RegExp _codeBlockStart = RegExp( + r'^\s*(?`{3,})(?\S+).*?$', +); + +final RegExp _splitArgs = RegExp( + r'(?[-\w]+)\s*(=\s*"(?.*?)"\s*)\s*', +); + +/// A code excerpt set or injection instruction +/// found in a file being processed. +sealed class _Instruction { + /// Parse and return an [_Instruction] from the specified [line]. + /// + /// The line is assumed to at least look like like an instruction. + /// + /// Errors are reported to the calling function + /// through the [reportError] callback. + static _Instruction fromLine( + String line, + Never Function(String error) reportError, + ) { + final match = _instructionPattern.firstMatch(line); + if (match == null) { + reportError('Invalid code excerpt syntax.'); + } + + final path = match.namedGroup('path'); + final argumentString = match.namedGroup('args')?.trim() ?? ''; + final argumentPairs = _splitArgs + .allMatches(argumentString) + .map((m) => (arg: m.namedGroup('arg')!, value: m.namedGroup('value')!)); + + if (path == null) { + if (argumentPairs.length != 1) { + reportError('A set instruction must have only one argument specified.'); + } + final argName = argumentPairs.first.arg; + final argValue = argumentPairs.first.value; + return switch (argName) { + 'path-base' => _SetPathBaseInstruction(argValue), + 'plaster' => _SetPlasterInstruction(argValue), + 'replace' => _SetFileReplaceInstruction( + stringToReplaceTransforms(argValue, reportError), + ), + _ => reportError( + 'A set instruction can only specify the ' + '`path-base`, `plaster`, and `replace` arguments.', + ), + }; + } + + return _InjectInstruction.fromArgs( + targetPath: path, + regionName: match.namedGroup('region') ?? '', + arguments: argumentPairs, + reportError: reportError, + ); + } +} + +sealed class _SetInstruction extends _Instruction {} + +final class _SetPathBaseInstruction extends _SetInstruction { + final String pathBase; + + _SetPathBaseInstruction(this.pathBase); +} + +final class _SetPlasterInstruction extends _SetInstruction { + final String plasterTemplate; + + _SetPlasterInstruction(this.plasterTemplate); +} + +final class _SetFileReplaceInstruction extends _SetInstruction { + final Iterable transforms; + + _SetFileReplaceInstruction(this.transforms); +} + +/// An excerpt instruction that indicates an injection of content +/// from a source file and specified region, +/// with optional configuration such as transforms applied. +final class _InjectInstruction extends _Instruction { + final String targetPath; + final String regionName; + + final List transforms; + + final int? indentBy; + final String? plasterTemplate; + + _InjectInstruction({ + required this.targetPath, + required this.regionName, + required this.transforms, + this.indentBy, + this.plasterTemplate, + }); + + factory _InjectInstruction.fromArgs({ + required String targetPath, + required String regionName, + required Iterable<({String arg, String value})> arguments, + required Never Function(String error) reportError, + }) { + String? indentByString; + String? plasterTemplate; + + final transforms = []; + + for (final (arg: argName, value: argValue) in arguments) { + switch (argName) { + case 'indent-by': + if (indentByString != null) { + reportError( + 'The `indent-by` argument can only be ' + 'specified once per instruction.', + ); + } + indentByString = argValue; + case 'plaster': + if (plasterTemplate != null) { + reportError( + 'The `plaster` argument can only be ' + 'specified once per instruction.', + ); + } + plasterTemplate = argValue; + case 'skip': + transforms.add(SkipTransform(int.parse(argValue))); + case 'take': + transforms.add(TakeTransform(int.parse(argValue))); + case 'from': + transforms.add(FromTransform(_argStringToPattern(argValue))); + case 'to': + transforms.add(ToTransform(_argStringToPattern(argValue))); + case 'remove': + transforms.add(RemoveTransform(_argStringToPattern(argValue))); + case 'retain': + transforms.add(RetainTransform(_argStringToPattern(argValue))); + case 'replace': + transforms.addAll(stringToReplaceTransforms(argValue, reportError)); + default: + reportError( + '$argName is an unsupported argument ' + 'in inject instructions.', + ); + } + } + + final indentBy = indentByString == null ? null : int.parse(indentByString); + + if (indentBy != null && indentBy < 0) { + reportError('The `indent-by` argument must be positive.'); + } + + return _InjectInstruction( + targetPath: targetPath, + regionName: regionName, + indentBy: indentBy, + plasterTemplate: plasterTemplate, + transforms: transforms, + ); + } +} + +/// Convert the specified string [value] to a regular expression +/// if wrapped in `/`. +Pattern _argStringToPattern(String value) { + if (value.startsWith('/') && value.endsWith('/')) { + final regularExpression = value.substring(1, value.length - 1); + return RegExp(regularExpression); + } + + // Unescape an escaped starting slash. + if (value.startsWith(r'\/')) { + return value.substring(1); + } + + return value; +} + +/// Convert from the specified [contentType], +/// representing a content type names or programming language, to +/// its line comment format, including prefix and suffix. +/// +/// `null` is returned if no line comment format is known. +({String prefix, String suffix})? _contentTypeToCommentFormat( + String contentType, +) { + switch (contentType.trim().toLowerCase()) { + case 'c': + case 'c++': + case 'cc': + case 'cpp': + case 'cs': + case 'csharp': + case 'dart': + case 'go': + case 'gradle': + case 'groovy': + case 'java': + case 'javascript': + case 'js': + case 'kotlin': + case 'kt': + case 'objc': + case 'rs': + case 'sass': + case 'scss': + case 'swift': + case 'ts': + case 'typescript': + return (prefix: '// ', suffix: ''); + case 'css': + return (prefix: '/* ', suffix: ' */'); + case 'html': + case 'xml': + return (prefix: ''); + case 'python': + case 'py': + case 'yml': + case 'yaml': + return (prefix: '# ', suffix: ''); + case _: + return null; + } +} diff --git a/pkgs/excerpter/lib/src/transform.dart b/pkgs/excerpter/lib/src/transform.dart new file mode 100644 index 00000000000..a72e2014fec --- /dev/null +++ b/pkgs/excerpter/lib/src/transform.dart @@ -0,0 +1,320 @@ +// Copyright (c) 2023. All rights reserved. Use of this source code +// is governed by a MIT-style license that can be found in the LICENSE file. + +import 'package:collection/collection.dart'; +import 'package:meta/meta.dart'; + +/// A stored transformation that can transform from +/// a collection of strings, usually lines, +/// to a transformed version using [transform]. +sealed class Transform { + /// Transform the provided [lines] with the transform + /// defined by this type. + @useResult + Iterable transform(Iterable lines); +} + +/// A base class for transforms based off a singular [Pattern]. +abstract final class PatternTransform extends Transform { + /// The pattern to apply during the transform. + final Pattern pattern; + + /// The super constructor for transforms that apply a singular pattern. + PatternTransform(this.pattern); +} + +/// A transform to remove the lines that don't contain the specified [pattern]. +final class RetainTransform extends PatternTransform { + /// Create a [RetainTransform] that removes the lines that + /// don't contain the specified [pattern]. + RetainTransform(super.pattern); + + @override + Iterable transform(Iterable lines) { + return lines.where((line) => line.contains(pattern)); + } +} + +/// A transform to remove the lines that contain the specified [pattern]. +final class RemoveTransform extends PatternTransform { + /// Create a [RemoveTransform] that removes the lines that + /// do contain the specified [pattern]. + RemoveTransform(super.pattern); + + @override + Iterable transform(Iterable lines) { + return lines.whereNot((line) => line.contains(pattern)); + } +} + +/// A transform to keep the lines after and including the first one +/// that contains the specified [pattern]. +final class FromTransform extends PatternTransform { + /// Create a [FromTransform] that keeps the lines after and including + /// the first one that contains the specified [pattern]. + FromTransform(super.pattern); + + @override + Iterable transform(Iterable lines) { + return lines.skipWhile((line) => !line.contains(pattern)); + } +} + +/// A transform to keep only the lines before and including the first one +/// that contains the specified [pattern]. +final class ToTransform extends PatternTransform { + /// Create a [ToTransform] that keeps the lines before and including + /// the first one that contains the specified [pattern]. + ToTransform(super.pattern); + + @override + Iterable transform(Iterable lines) { + final newLines = []; + for (final line in lines) { + newLines.add(line); + if (line.contains(pattern)) { + break; + } + } + return newLines; + } +} + +/// A base class for transforms based off a singular integer [count]. +abstract final class AmountTransform extends Transform { + /// The count to use during the transform. + final int count; + + /// The super constructor for transforms that utilize a singular [count]. + AmountTransform(this.count); +} + +/// A transform to skip the first [count] lines if positive or +/// the last [count] lines if negative. +final class SkipTransform extends AmountTransform { + /// Create a [SkipTransform] that skips the first [count] lines if + /// positive or the last [count] lines if negative. + SkipTransform(super.count); + + @override + Iterable transform(Iterable lines) { + if (count.isNegative) { + return lines.take(lines.length + count); + } else { + return lines.skip(count); + } + } +} + +/// A transform to keep the first [count] lines if positive or +/// the last [count] lines if negative. +final class TakeTransform extends AmountTransform { + /// Create a [TakeTransform] that keeps the first [count] lines if + /// positive or the last [count] lines if negative. + TakeTransform(super.count); + + @override + Iterable transform(Iterable lines) { + if (count.isNegative) { + return lines.skip(lines.length + count); + } else { + return lines.take(count); + } + } +} + +/// A transform to indent each line with [count] whitespace. +final class IndentTransform extends AmountTransform { + /// Create a [IndentTransform] that indents each line with [count] whitespace. + IndentTransform(super.count) : assert(count >= 0); + + @override + Iterable transform(Iterable lines) { + if (count == 0) return lines; + + final indentString = ' ' * count; + return lines.map((line) => '$indentString$line'); + } +} + +/// Convert the [replaceInstructions] string that can include multiple +/// regular expression-based replacements to multiple +/// [ReplaceTransform] instances representing their transformation. +/// +/// Errors in parsing the [replaceInstructions] are reported to the +/// calling function through the [reportError] callback. +@useResult +Iterable stringToReplaceTransforms( + String replaceInstructions, + Never Function(String) reportError, +) { + final parts = replaceInstructions + .replaceAll(r'\/', _placeholderString) + .split('/') + .map((part) => part.replaceAll(_placeholderString, '/')) + .toList(growable: false); + + final length = parts.length; + if (length < 4 || length % 3 != 1) { + reportError('Each replace transform must have 3 parts.'); + } + + final start = parts[0]; + if (start.isNotEmpty) { + reportError('A replace transform must start with a forward slash (`/`).'); + } + + final transforms = []; + + for (var index = 1; index < length; index += 3) { + final end = parts[index + 2]; + if (!_endReplacePattern.hasMatch(end)) { + reportError('A replace transform must end with `g;`.'); + } + + final originalPattern = parts[index]; + final replaceWith = parts[index + 1]; + final encodedReplaceWith = _encodeSlashChar(replaceWith); + + if (!encodedReplaceWith.contains(_matchDollarNumRE)) { + transforms.add( + SimpleReplaceTransform( + RegExp(originalPattern, multiLine: true), + encodedReplaceWith, + ), + ); + } else { + transforms.add( + BackReferenceReplaceTransform( + RegExp(originalPattern, multiLine: true), + encodedReplaceWith, + ), + ); + } + } + + return transforms; +} + +/// A base class for replacement transforms that convert +/// text from one form to another, based on a provided [from] pattern +/// and [to] string. +sealed class ReplaceTransform extends Transform { + /// The pattern to match text to consider for replacement. + final Pattern from; + + /// The string to replace the text matching the [from] pattern with. + final String to; + + /// The super constructor for transforms that apply a replacement. + ReplaceTransform(this.from, this.to); + + /// Transform the provided [lines] with the transform + /// defined by this type. + /// + /// Note that, compared to other [Transform] types, + /// [ReplaceTransform] subtypes tend to operator on combined lines + /// rather than each line individually. + @override + Iterable transform(Iterable lines); +} + +/// A transform to replace each instance of the +/// [from] pattern with the [to] string. +final class SimpleReplaceTransform extends ReplaceTransform { + /// Create a [SimpleReplaceTransform] that replaces each instance of the + /// [from] pattern with the [to] string in the combined text. + SimpleReplaceTransform(super.from, super.to); + + @override + Iterable transform(Iterable lines) { + return lines.join('\n').replaceAll(from, to).split('\n'); + } +} + +/// A transform to replace each instance of the +/// [from] pattern with the [to] string while +/// allowing backreferences to captured groups in the [to] string. +final class BackReferenceReplaceTransform extends ReplaceTransform { + /// Create a [BackReferenceReplaceTransform] that replaces each instance of + /// the [from] pattern with the [to] string in the combined text, + /// while allowing backreferences to captured groups in the [to] string. + BackReferenceReplaceTransform(super.from, super.to); + + @override + Iterable transform(Iterable lines) { + return lines + .join('\n') + .replaceAllMapped( + from, + (match) => to.replaceAllMapped(_matchDollarNumRE, (replaceMatch) { + // The following works to match JS `replace` semantics. + // $$ becomes $ in a replacement string. + + final dollarSignCount = replaceMatch[1]!.length; + + // The escaped dollar sign characters present (if any). + final escapedDollarSigns = r'$' * (dollarSignCount ~/ 2); + + // Potentially a reference to a captured group, + // otherwise the content after the escaped dollar signs. + final potentialGroupReference = replaceMatch[2]; + + if (potentialGroupReference == null || + potentialGroupReference.isEmpty) { + return escapedDollarSigns; + } + + if (dollarSignCount.isEven) { + return '$escapedDollarSigns$potentialGroupReference'; + } + + // $& references the entire matched substring. + if (potentialGroupReference == '&') { + return '$escapedDollarSigns${match[0]}'; + } + + final groupNumber = int.tryParse(potentialGroupReference); + if (groupNumber == null || groupNumber > match.groupCount) { + // If there is no corresponding capture group, + // just output the reference itself. + return '$escapedDollarSigns\$$potentialGroupReference'; + } + + return '$escapedDollarSigns${match[groupNumber]}'; + }), + ) + .split('\n'); + } +} + +const String _placeholderString = '\u{0}'; + +final RegExp _matchDollarNumRE = RegExp(r'(\$+)(&|\d*)'); + +final RegExp _slashHexCharRE = RegExp(r'\\x(..)'); +final RegExp _slashLetterRE = RegExp(r'\\([\\nt])'); + +/// Encode special characters: '\t', `\n`, and `\xHH` where `HH` are hex digits. +String _encodeSlashChar(String s) => s + .replaceAllMapped(_slashLetterRE, (match) => _slashCharToChar(match[1])) + .replaceAllMapped( + _slashHexCharRE, + // At this point, escaped `\` is encoded as [_placeholderString]. + (match) => _hexToChar(match[1], errorValue: '\\x${match[1]}'), + ) + .replaceAll(_placeholderString, '\\'); // Recover `\` characters. + +String _hexToChar(String? hexDigits, {required String errorValue}) { + final charCode = int.tryParse(hexDigits ?? '', radix: 16); + return charCode == null ? errorValue : String.fromCharCode(charCode); +} + +String _slashCharToChar(String? char) => switch (char) { + 'n' => '\n', + 't' => '\t', + '\\' => _placeholderString, + _ => '\\$char', +}; + +final RegExp _endReplacePattern = RegExp(r'^g;?\s*$'); diff --git a/pkgs/excerpter/lib/src/update.dart b/pkgs/excerpter/lib/src/update.dart new file mode 100644 index 00000000000..8f8b342e49d --- /dev/null +++ b/pkgs/excerpter/lib/src/update.dart @@ -0,0 +1,190 @@ +// Copyright (c) 2023. All rights reserved. Use of this source code +// is governed by a MIT-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as path; + +import 'inject.dart'; +import 'transform.dart'; + +/// An excerpt updater that can run over the files with [validTargetExtensions] +/// in the specified paths with [update], and updates their code excerpts +/// according to their injection instructions. +final class Updater { + /// A pattern to check for paths to ignore or not descend in to. + static final RegExp _ignoreHiddenPathsPattern = RegExp(r'(^|/)\..*($|/)'); + + /// The transforms to run on each retrieved region. + final Iterable defaultTransforms; + + /// Patterns of paths to avoid updating. + final Iterable excludePaths; + + /// Extensions of files to consider for injection updating. + final Set validTargetExtensions; + + /// The path to the directory of the source files to pull regions from. + final String baseSourcePath; + + /// The content to include in plaster replacements by default. + final String defaultPlasterContent; + + /// Whether to continue updating or checking for updates if + /// an error is experienced. + final bool continueOnError; + + /// Create an [Updater] that can be used to [update] a collection of files + /// with the specified [validTargetExtensions] but + /// not matching [excludePaths]. + /// + /// Regions in injection instructions are pulled from files + /// within the [baseSourcePath]. + /// + /// If [withDefaultExclusions] is `true`, files and directories + /// hidden by starting with a `.` are excluded as well. + /// + /// The [defaultTransforms] specified are run on each region + /// before transforms provided by instructions on the page. + Updater({ + required this.baseSourcePath, + required this.validTargetExtensions, + this.continueOnError = false, + this.defaultPlasterContent = '...', + this.defaultTransforms = const [], + Iterable excludePaths = const [], + bool withDefaultExclusions = true, + }) : excludePaths = [ + if (withDefaultExclusions) _ignoreHiddenPathsPattern, + ...excludePaths, + ]; + + /// Use the configuration of this [Updater] to process the + /// file at the [pathToUpdate], optionally making updates + /// if [makeUpdates] is `true`. + @useResult + Future update( + String pathToUpdate, { + bool makeUpdates = true, + }) async { + final pathsToUpdate = await _findTargetFiles(pathToUpdate); + final warnings = []; + var excerptsVisited = 0; + var excerptsNeedingUpdates = 0; + var filesVisited = 0; + var madeUpdates = false; + final errors = []; + + for (final currentPath in pathsToUpdate) { + final fileUpdater = FileUpdater( + currentPath, + baseSourcePath: baseSourcePath, + defaultPlasterContent: defaultPlasterContent, + defaultTransforms: defaultTransforms, + ); + + try { + final results = await fileUpdater.process(); + warnings.addAll(results.warnings); + excerptsVisited += results.excerptsVisited; + excerptsNeedingUpdates += results.excerptsUpdated.length; + + if (makeUpdates && results.needsUpdates) { + madeUpdates = true; + await results.writeUpdates(); + } + + filesVisited += 1; + } on InjectionException catch (e) { + errors.add(e.toString()); + if (!continueOnError) { + break; + } + } + } + + return ExcerptUpdateResult._( + excerptsVisited: excerptsVisited, + excerptsNeedingUpdates: excerptsNeedingUpdates, + madeUpdates: madeUpdates, + filesVisited: filesVisited, + totalFilesToVisit: pathsToUpdate.length, + warnings: warnings, + errors: errors, + ); + } + + @useResult + Future> _findTargetFiles(String pathToEntity) async { + if (_shouldExclude(pathToEntity)) return const []; + + final entityType = await FileSystemEntity.type(pathToEntity); + if (entityType == FileSystemEntityType.directory) { + return [ + await for (final entity in Directory( + pathToEntity, + ).list(followLinks: false)) + ...await _findTargetFiles(entity.path), + ]; + } else if (entityType == FileSystemEntityType.file && + validTargetExtensions.contains(path.extension(pathToEntity))) { + return [pathToEntity]; + } + + return const []; + } + + bool _shouldExclude(String path) => + excludePaths.any((excludePattern) => path.contains(excludePattern)); +} + +/// The result of running a [Updater] on a collection of files. +@immutable +final class ExcerptUpdateResult { + /// The non-fatal warnings experienced when + /// processing the specified collection of files. + final Iterable warnings; + + /// The fatal errors experienced when + /// processing the specified collection of files. + final Iterable errors; + + /// The amount of code excerpt injections visited when + /// processing the specified collection of files. + /// + /// Note this may not be all excerpts in the collection + /// as a fatal error may have been experienced. + final int excerptsVisited; + + /// The amount of code excerpt injections that needed updates when + /// processing the specified collection of files. + /// + /// Note this may not be all excerpts needing updates in the collection + /// as a fatal error may have been experienced. + final int excerptsNeedingUpdates; + + /// The amount of files visited successfully without errors when + /// processing the specified collection of files. + /// + /// If this differs from [totalFilesToVisit], that means + /// there was a fatal error during processing. + final int filesVisited; + + /// The total amount of files in the specified collection of files + /// that were meant to be visited. + final int totalFilesToVisit; + + /// If the [Updater] wrote updates to the files needing them. + final bool madeUpdates; + + ExcerptUpdateResult._({ + required this.excerptsVisited, + required this.excerptsNeedingUpdates, + required this.filesVisited, + required this.totalFilesToVisit, + required this.madeUpdates, + this.warnings = const [], + this.errors = const [], + }); +} diff --git a/pkgs/excerpter/pubspec.yaml b/pkgs/excerpter/pubspec.yaml new file mode 100644 index 00000000000..9c4208deabf --- /dev/null +++ b/pkgs/excerpter/pubspec.yaml @@ -0,0 +1,25 @@ +name: excerpter +description: Update code excerpts in Markdown documentation from source files. +version: 0.1.0 +publish_to: none + +resolution: workspace +environment: + sdk: ^3.11.0 + +dependencies: + args: ^2.6.0 + collection: ^1.19.0 + file: ^7.0.0 + glob: ^2.1.2 + meta: ^1.16.0 + path: ^1.9.0 + +dev_dependencies: + analysis_defaults: + path: ../analysis_defaults + io: ^1.0.5 + test: ^1.28.0 + +executables: + excerpter: excerpter diff --git a/pkgs/excerpter/test/cli_test.dart b/pkgs/excerpter/test/cli_test.dart new file mode 100644 index 00000000000..4a729e6fa72 --- /dev/null +++ b/pkgs/excerpter/test/cli_test.dart @@ -0,0 +1,30 @@ +// Copyright (c) 2023. All rights reserved. Use of this source code +// is governed by a MIT-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:path/path.dart' as path; +import 'package:test/test.dart'; + +void main() { + final excerpterPath = path.join('bin', 'excerpter.dart'); + + test('no args', () { + final process = Process.runSync(Platform.executable, [ + 'run', + excerpterPath, + ]); + + expect(process.exitCode, equals(1)); + }); + + test('invalid format', () { + final process = Process.runSync(Platform.executable, [ + 'run', + excerpterPath, + '-f-23-423-4', + ]); + + expect(process.exitCode, equals(1)); + }); +} diff --git a/pkgs/excerpter/test/transform_test.dart b/pkgs/excerpter/test/transform_test.dart new file mode 100644 index 00000000000..dfdc50d74f0 --- /dev/null +++ b/pkgs/excerpter/test/transform_test.dart @@ -0,0 +1,242 @@ +// Copyright (c) 2023. All rights reserved. Use of this source code +// is governed by a MIT-style license that can be found in the LICENSE file. + +import 'package:excerpter/excerpter.dart'; +import 'package:test/test.dart'; + +void main() { + group('Pattern transforms', _patternTransforms); + group('Amount transforms', _amountTransforms); + group('Replace transforms', _replaceTransforms); + group('String to replace transforms', _stringToReplaceTransforms); +} + +void _patternTransforms() { + test('retain all', () { + final all = ['aaa', 'aabb', 'abc', 'aacc']; + expect(RetainTransform('a').transform(all), orderedEquals(all)); + }); + + test('retain some', () { + expect( + RetainTransform('b').transform(['aaa', 'aabb', 'abc', 'cccc']), + orderedEquals(['aabb', 'abc']), + ); + }); + + test('retain none', () { + expect( + RetainTransform('d').transform(['aaa', 'aabb', 'abc', 'cccc']), + orderedEquals([]), + ); + }); + + test('remove all', () { + final all = ['aaa', 'aabb', 'abc', 'aacc']; + expect(RemoveTransform('a').transform(all), orderedEquals([])); + }); + + test('remove some', () { + expect( + RemoveTransform('b').transform(['aaa', 'aabb', 'abc', 'cccc']), + orderedEquals(['aaa', 'cccc']), + ); + }); + + test('remove none', () { + final all = ['aaa', 'aabb', 'abc', 'cccc']; + expect(RemoveTransform('d').transform(all), orderedEquals(all)); + }); + + test('from all', () { + final all = ['aaa', 'aabb', 'abc', 'aacc']; + expect(FromTransform('aaa').transform(all), orderedEquals(all)); + }); + + test('from some', () { + expect( + FromTransform('abc').transform(['aaa', 'aabb', 'abc', 'cccc']), + orderedEquals(['abc', 'cccc']), + ); + }); + + test('from none', () { + expect( + FromTransform('d').transform(['aaa', 'aabb', 'abc', 'cccc']), + orderedEquals([]), + ); + }); + + test('to all', () { + final all = ['aaa', 'aabb', 'abc', 'aacc']; + expect(ToTransform('aacc').transform(all), orderedEquals(all)); + }); + + test('to some', () { + expect( + ToTransform('aabb').transform(['aaa', 'aabb', 'abc', 'aacc']), + orderedEquals(['aaa', 'aabb']), + ); + }); + + test('to none', () { + final all = ['aaa', 'aabb', 'abc', 'cccc']; + expect(ToTransform('d').transform(all), orderedEquals(all)); + }); +} + +void _amountTransforms() { + test('skip negative', () { + expect( + SkipTransform(-2).transform(['aaa', 'aabb', 'abc', 'cccc']), + orderedEquals(['aaa', 'aabb']), + ); + }); + + test('skip zero', () { + final all = ['aaa', 'aabb', 'abc', 'cccc']; + expect(SkipTransform(0).transform(all), orderedEquals(all)); + }); + + test('skip positive', () { + expect( + SkipTransform(2).transform(['aaa', 'aabb', 'abc', 'cccc']), + orderedEquals(['abc', 'cccc']), + ); + }); + + test('skip all', () { + expect( + SkipTransform(4).transform(['aaa', 'aabb', 'abc', 'cccc']), + orderedEquals([]), + ); + }); + + test('take negative', () { + expect( + TakeTransform(-2).transform(['aaa', 'aabb', 'abc', 'cccc']), + orderedEquals(['abc', 'cccc']), + ); + }); + + test('take zero', () { + expect( + TakeTransform(0).transform(['aaa', 'aabb', 'abc', 'cccc']), + orderedEquals([]), + ); + }); + + test('take positive', () { + expect( + TakeTransform(2).transform(['aaa', 'aabb', 'abc', 'cccc']), + orderedEquals(['aaa', 'aabb']), + ); + }); + + test('take all', () { + final all = ['aaa', 'aabb', 'abc', 'cccc']; + expect(TakeTransform(4).transform(all), orderedEquals(all)); + }); + + test('indent negative', () { + expect(() => IndentTransform(-2), throwsA(isA())); + }); + + test('indent zero', () { + final all = ['a', ' b', ' c']; + expect(IndentTransform(0).transform(all), orderedEquals(all)); + }); + + test('indent positive', () { + expect( + IndentTransform(1).transform(['a', ' b', ' c']), + orderedEquals([' a', ' b', ' c']), + ); + }); +} + +void _replaceTransforms() { + test('replace simple some', () { + expect( + SimpleReplaceTransform( + RegExp('Hello'), + 'Halo', + ).transform(['Hello world!!', 'Bye!']), + orderedEquals(['Halo world!!', 'Bye!']), + ); + }); + + test('replace simple split', () { + expect( + SimpleReplaceTransform( + RegExp('Hi\nDash'), + 'Bye\nFriends', + ).transform(['Hi', 'Dash!']), + orderedEquals(['Bye', 'Friends!']), + ); + }); + + test('replace backreferences single capture group', () { + expect( + BackReferenceReplaceTransform( + RegExp('(Hello )Dash'), + r'$1World', + ).transform(['Hello Dash']), + orderedEquals(['Hello World']), + ); + }); + + test('replace backreferences entire captured', () { + expect( + BackReferenceReplaceTransform( + RegExp('Hello Dash'), + r'[!$&!]', + ).transform(['Hello Dash, you are very blue.']), + orderedEquals(['[!Hello Dash!], you are very blue.']), + ); + }); +} + +void _stringToReplaceTransforms() { + Never errorNotExpected(String error) { + fail('Error not expected - $error'); + } + + Never errorExpected(String error) { + throw _ExpectedException(); + } + + test('empty', () { + expect( + () => stringToReplaceTransforms('', errorExpected), + throwsA(isA<_ExpectedException>()), + ); + }); + + test('missing ending', () { + expect( + () => stringToReplaceTransforms('/Hello/Halo/', errorExpected), + throwsA(isA<_ExpectedException>()), + ); + }); + + test('single replace', () { + final simpleReplace = stringToReplaceTransforms( + '/Hello/Hi/g;', + errorNotExpected, + ); + expect(simpleReplace.length, equals(1)); + expect(simpleReplace.first.from, equals(RegExp('Hello', multiLine: true))); + expect(simpleReplace.first.to, equals('Hi')); + }); + + test('multiple replace', () { + final multipleReplace = stringToReplaceTransforms( + '/Hello/Hi/g;/World/Dash/g;', + errorNotExpected, + ); + expect(multipleReplace.length, equals(2)); + }); +} + +final class _ExpectedException implements Exception {} diff --git a/pkgs/excerpter/test/updater_test.dart b/pkgs/excerpter/test/updater_test.dart new file mode 100644 index 00000000000..a21b07e6dcb --- /dev/null +++ b/pkgs/excerpter/test/updater_test.dart @@ -0,0 +1,86 @@ +// Copyright (c) 2023. All rights reserved. Use of this source code +// is governed by a MIT-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:excerpter/excerpter.dart'; +import 'package:io/io.dart'; +import 'package:path/path.dart' as path; +import 'package:test/test.dart'; + +void main() { + group('Updater setup', _setup); + group('Default update behavior', _defaultBehavior); +} + +void _setup() { + test('includes default exclusions', () { + expect( + Updater(baseSourcePath: 'example', validTargetExtensions: {'.md'}) + .excludePaths + .any((excludePattern) => excludePattern.hasMatch('.dart_tool')), + isTrue, + ); + }); + + test('includes default exclusions', () { + expect( + Updater(baseSourcePath: 'example', validTargetExtensions: {'.md'}) + .excludePaths + .any((excludePattern) => excludePattern.hasMatch('.dart_tool')), + isTrue, + ); + }); +} + +void _defaultBehavior() { + final examplePath = path.join('test_data', 'example'); + final expectedPath = path.join('test_data', 'expected'); + final srcPath = path.join('test_data', 'src'); + final srcCopyPath = path.join('test_data', 'src_copy'); + + final updater = Updater( + baseSourcePath: examplePath, + validTargetExtensions: {'.md'}, + ); + + test('no updates - no errors - no update', () async { + final results = await updater.update(expectedPath, makeUpdates: false); + expect(results.warnings, hasLength(0)); + expect(results.errors, hasLength(0)); + expect(results.excerptsNeedingUpdates, equals(0)); + expect(results.excerptsVisited, greaterThan(0)); + expect(results.totalFilesToVisit, equals(4)); + expect(results.filesVisited, equals(4)); + expect(results.madeUpdates, isFalse); + }); + + test('update works', () async { + copyPathSync(srcPath, srcCopyPath); + + final results = await updater.update(srcCopyPath, makeUpdates: true); + expect(results.warnings, hasLength(0)); + expect(results.errors, hasLength(0)); + expect(results.excerptsNeedingUpdates, greaterThan(0)); + expect(results.excerptsNeedingUpdates, equals(results.excerptsVisited)); + expect(results.madeUpdates, isTrue); + + for (final expectedFile in Directory(expectedPath).listSync()) { + if (expectedFile is File) { + final updatedPath = path.join( + srcCopyPath, + path.basename(expectedFile.path), + ); + final updatedFile = File(updatedPath); + expect(updatedFile.existsSync(), isTrue); + + expect( + expectedFile.readAsStringSync(), + equals(updatedFile.readAsStringSync()), + ); + } + } + + Directory(srcCopyPath).deleteSync(recursive: true); + }); +} diff --git a/pkgs/excerpter/test_data/example/dartdoc.dart b/pkgs/excerpter/test_data/example/dartdoc.dart new file mode 100644 index 00000000000..9ef0625dbb5 --- /dev/null +++ b/pkgs/excerpter/test_data/example/dartdoc.dart @@ -0,0 +1,4 @@ +/// ```html +///

HTML is magical!

+/// ``` +class HTML {} diff --git a/pkgs/excerpter/test_data/example/plaster.dart b/pkgs/excerpter/test_data/example/plaster.dart new file mode 100644 index 00000000000..d9054f7c89b --- /dev/null +++ b/pkgs/excerpter/test_data/example/plaster.dart @@ -0,0 +1,44 @@ +// #docregion single +void single() { + // #enddocregion single + print('Not showing up!'); + // #docregion single +} +// #enddocregion single + +// #docregion multiple +void multiple() { + // #enddocregion multiple + print('Not showing up!'); + // #docregion multiple + print('Showing up!'); + // #enddocregion multiple + print('Not showing up!'); + // #docregion multiple +} +// #enddocregion multiple + +// #docregion remove +void remove() { + // #enddocregion remove + print('Nothing here.'); + // #docregion remove +} +// #enddocregion remove + +// #docregion custom +void custom() { + // #enddocregion custom + print('Different plaster here.'); + // #docregion custom +} +// #enddocregion custom + +// #docregion template +void template() { + // #enddocregion template + print('Templated plaster here.'); + // #docregion template +} + +// #enddocregion template diff --git a/pkgs/excerpter/test_data/example/simple_main.dart b/pkgs/excerpter/test_data/example/simple_main.dart new file mode 100644 index 00000000000..f4c4774e41e --- /dev/null +++ b/pkgs/excerpter/test_data/example/simple_main.dart @@ -0,0 +1,3 @@ +void main() { + print('Hello world!'); +} diff --git a/pkgs/excerpter/test_data/example/simple_region.dart b/pkgs/excerpter/test_data/example/simple_region.dart new file mode 100644 index 00000000000..bb04da53306 --- /dev/null +++ b/pkgs/excerpter/test_data/example/simple_region.dart @@ -0,0 +1,5 @@ +void main() { + // #docregion hello + print('Hello world!'); + // #enddocregion hello +} diff --git a/pkgs/excerpter/test_data/example/transforms.dart b/pkgs/excerpter/test_data/example/transforms.dart new file mode 100644 index 00000000000..4b1f5b573b5 --- /dev/null +++ b/pkgs/excerpter/test_data/example/transforms.dart @@ -0,0 +1,6 @@ +// #docregion indent +void indent() { + print('indent'); +} + +// #enddocregion indent diff --git a/pkgs/excerpter/test_data/expected/dartdoc.md b/pkgs/excerpter/test_data/expected/dartdoc.md new file mode 100644 index 00000000000..9e98cd711b6 --- /dev/null +++ b/pkgs/excerpter/test_data/expected/dartdoc.md @@ -0,0 +1,12 @@ +## API docs + +Markdown code block opened and closed with more than 3 backticks +that contains another Markdown code block declared with just 3. + + +````dart +/// ```html +///

HTML is magical!

+/// ``` +class HTML {} +```` diff --git a/pkgs/excerpter/test_data/expected/plaster.md b/pkgs/excerpter/test_data/expected/plaster.md new file mode 100644 index 00000000000..41b04762152 --- /dev/null +++ b/pkgs/excerpter/test_data/expected/plaster.md @@ -0,0 +1,46 @@ +## Test plaster feature + +### Single plaster + + +```dart +void single() { + // ... +} +``` + +### Multiple plaster + + +```dart +void multiple() { + // ... + print('Showing up!'); + // ... +} +``` + +### Remove plaster + + +```dart +void remove() { +} +``` + +### Custom template + + +```dart +void custom() { + /*...*/ +} +``` + + +```dart +void template() { + /* ... */ +} + +``` diff --git a/pkgs/excerpter/test_data/expected/simple.md b/pkgs/excerpter/test_data/expected/simple.md new file mode 100644 index 00000000000..6cd57b20d7e --- /dev/null +++ b/pkgs/excerpter/test_data/expected/simple.md @@ -0,0 +1,17 @@ +## Simple usages + +Simple excerpt with no region: + + +```dart +void main() { + print('Hello world!'); +} +``` + +Simple excerpt with region: + + +```dart +print('Hello world!'); +``` diff --git a/pkgs/excerpter/test_data/expected/transforms.md b/pkgs/excerpter/test_data/expected/transforms.md new file mode 100644 index 00000000000..bc7b9130285 --- /dev/null +++ b/pkgs/excerpter/test_data/expected/transforms.md @@ -0,0 +1,11 @@ +## Transformations + +### Indentation + + +```dart + void indent() { + print('indent'); + } + +``` diff --git a/pkgs/excerpter/test_data/src/dartdoc.md b/pkgs/excerpter/test_data/src/dartdoc.md new file mode 100644 index 00000000000..c23b80ea5ec --- /dev/null +++ b/pkgs/excerpter/test_data/src/dartdoc.md @@ -0,0 +1,8 @@ +## API docs + +Markdown code block opened and closed with more than 3 backticks +that contains another Markdown code block declared with just 3. + + +````dart +```` diff --git a/pkgs/excerpter/test_data/src/plaster.md b/pkgs/excerpter/test_data/src/plaster.md new file mode 100644 index 00000000000..4d6bfaad3fc --- /dev/null +++ b/pkgs/excerpter/test_data/src/plaster.md @@ -0,0 +1,29 @@ +## Test plaster feature + +### Single plaster + + +```dart +``` + +### Multiple plaster + + +```dart +``` + +### Remove plaster + + +```dart +``` + +### Custom template + + +```dart +``` + + +```dart +``` diff --git a/pkgs/excerpter/test_data/src/simple.md b/pkgs/excerpter/test_data/src/simple.md new file mode 100644 index 00000000000..45288b92c13 --- /dev/null +++ b/pkgs/excerpter/test_data/src/simple.md @@ -0,0 +1,13 @@ +## Simple usages + +Simple excerpt with no region: + + +```dart +``` + +Simple excerpt with region: + + +```dart +``` diff --git a/pkgs/excerpter/test_data/src/transforms.md b/pkgs/excerpter/test_data/src/transforms.md new file mode 100644 index 00000000000..af7e872dfdd --- /dev/null +++ b/pkgs/excerpter/test_data/src/transforms.md @@ -0,0 +1,7 @@ +## Transformations + +### Indentation + + +```dart +``` diff --git a/pubspec.yaml b/pubspec.yaml index 39826b981e5..afcb53f3ac6 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -5,5 +5,7 @@ environment: sdk: ^3.11.0 workspace: + - pkgs/analysis_defaults + - pkgs/excerpter - site - tool/dash_site diff --git a/site/pubspec.yaml b/site/pubspec.yaml index 487797b18ec..026183a31d8 100644 --- a/site/pubspec.yaml +++ b/site/pubspec.yaml @@ -29,10 +29,7 @@ dependencies: dev_dependencies: analysis_defaults: - git: - url: https://github.com/dart-lang/site-shared - path: pkgs/analysis_defaults - ref: f91ed8ecef6a0b31685804fe4102b25fda021460 + path: ../pkgs/analysis_defaults build_runner: ^2.11.0 build_web_compilers: ^4.4.9 jaspr_builder: ^0.22.3 diff --git a/tool/dash_site/lib/src/commands/analyze_dart.dart b/tool/dash_site/lib/src/commands/analyze_dart.dart index af942ad03e6..660e0211e11 100644 --- a/tool/dash_site/lib/src/commands/analyze_dart.dart +++ b/tool/dash_site/lib/src/commands/analyze_dart.dart @@ -33,9 +33,10 @@ final class AnalyzeDartCommand extends Command { int analyzeDart({bool verboseLogging = false}) { final directoriesToAnalyze = [ - path.join('site'), + 'examples', + 'pkgs', + 'site', path.join('tool', 'dash_site'), - path.join('examples'), ]; if (!verboseLogging) { @@ -52,7 +53,7 @@ int analyzeDart({bool verboseLogging = false}) { return pubGetResult; } - final flutterAnalyzeOutput = Process.runSync('flutter', const [ + final flutterAnalyzeOutput = Process.runSync('dart', const [ 'analyze', '.', ], workingDirectory: directory); diff --git a/tool/dash_site/lib/src/commands/format_dart.dart b/tool/dash_site/lib/src/commands/format_dart.dart index 468d7ff0664..4403e994dec 100644 --- a/tool/dash_site/lib/src/commands/format_dart.dart +++ b/tool/dash_site/lib/src/commands/format_dart.dart @@ -5,7 +5,6 @@ import 'dart:io'; import 'package:args/command_runner.dart'; -import 'package:path/path.dart' as path; import '../utils.dart'; @@ -34,17 +33,7 @@ final class FormatDartCommand extends Command { } int formatDart({bool justCheck = false}) { - // Currently format all Dart files in the /tool directory - // and everything in /examples. - final directoriesToFormat = [ - 'site', - 'tool', - ...Directory('examples') - .listSync() - .whereType() - .map((e) => e.path) - .where((e) => !path.basename(e).startsWith('.')), - ]; + final directoriesToFormat = ['examples', 'pkgs', 'site', 'tool']; final dartFormatOutput = Process.runSync(Platform.resolvedExecutable, [ 'format', diff --git a/tool/dash_site/pubspec.yaml b/tool/dash_site/pubspec.yaml index 3ec572948af..3332728de35 100644 --- a/tool/dash_site/pubspec.yaml +++ b/tool/dash_site/pubspec.yaml @@ -9,17 +9,11 @@ environment: dependencies: args: ^2.7.0 excerpter: - git: - url: https://github.com/dart-lang/site-shared - path: pkgs/excerpter - ref: f91ed8ecef6a0b31685804fe4102b25fda021460 + path: ../../pkgs/excerpter io: ^1.0.5 linkcheck: ^3.1.0 path: ^1.9.1 dev_dependencies: analysis_defaults: - git: - url: https://github.com/dart-lang/site-shared - path: pkgs/analysis_defaults - ref: f91ed8ecef6a0b31685804fe4102b25fda021460 + path: ../../pkgs/analysis_defaults From fa6d18ec0156d0d1f970f68ae90d8992cc8dd81d Mon Sep 17 00:00:00 2001 From: Parker Lougheed Date: Sat, 21 Mar 2026 18:54:06 +0800 Subject: [PATCH 2/4] Rename directory to /packages --- README.md | 2 +- {pkgs => packages}/analysis_defaults/analysis_options.yaml | 0 {pkgs => packages}/analysis_defaults/lib/analysis.yaml | 0 {pkgs => packages}/analysis_defaults/pubspec.yaml | 0 {pkgs => packages}/excerpter/LICENSE | 0 {pkgs => packages}/excerpter/README.md | 0 {pkgs => packages}/excerpter/analysis_options.yaml | 0 {pkgs => packages}/excerpter/bin/excerpter.dart | 0 {pkgs => packages}/excerpter/lib/excerpter.dart | 0 {pkgs => packages}/excerpter/lib/src/extract.dart | 0 {pkgs => packages}/excerpter/lib/src/inject.dart | 0 {pkgs => packages}/excerpter/lib/src/transform.dart | 0 {pkgs => packages}/excerpter/lib/src/update.dart | 0 {pkgs => packages}/excerpter/pubspec.yaml | 0 {pkgs => packages}/excerpter/test/cli_test.dart | 0 {pkgs => packages}/excerpter/test/transform_test.dart | 0 {pkgs => packages}/excerpter/test/updater_test.dart | 0 {pkgs => packages}/excerpter/test_data/example/dartdoc.dart | 0 {pkgs => packages}/excerpter/test_data/example/plaster.dart | 0 .../excerpter/test_data/example/simple_main.dart | 0 .../excerpter/test_data/example/simple_region.dart | 0 .../excerpter/test_data/example/transforms.dart | 0 {pkgs => packages}/excerpter/test_data/expected/dartdoc.md | 0 {pkgs => packages}/excerpter/test_data/expected/plaster.md | 0 {pkgs => packages}/excerpter/test_data/expected/simple.md | 0 {pkgs => packages}/excerpter/test_data/expected/transforms.md | 0 {pkgs => packages}/excerpter/test_data/src/dartdoc.md | 0 {pkgs => packages}/excerpter/test_data/src/plaster.md | 0 {pkgs => packages}/excerpter/test_data/src/simple.md | 0 {pkgs => packages}/excerpter/test_data/src/transforms.md | 0 pubspec.yaml | 4 ++-- site/pubspec.yaml | 2 +- src/content/contribute/docs/excerpts.md | 2 +- tool/dash_site/pubspec.yaml | 4 ++-- 34 files changed, 7 insertions(+), 7 deletions(-) rename {pkgs => packages}/analysis_defaults/analysis_options.yaml (100%) rename {pkgs => packages}/analysis_defaults/lib/analysis.yaml (100%) rename {pkgs => packages}/analysis_defaults/pubspec.yaml (100%) rename {pkgs => packages}/excerpter/LICENSE (100%) rename {pkgs => packages}/excerpter/README.md (100%) rename {pkgs => packages}/excerpter/analysis_options.yaml (100%) rename {pkgs => packages}/excerpter/bin/excerpter.dart (100%) rename {pkgs => packages}/excerpter/lib/excerpter.dart (100%) rename {pkgs => packages}/excerpter/lib/src/extract.dart (100%) rename {pkgs => packages}/excerpter/lib/src/inject.dart (100%) rename {pkgs => packages}/excerpter/lib/src/transform.dart (100%) rename {pkgs => packages}/excerpter/lib/src/update.dart (100%) rename {pkgs => packages}/excerpter/pubspec.yaml (100%) rename {pkgs => packages}/excerpter/test/cli_test.dart (100%) rename {pkgs => packages}/excerpter/test/transform_test.dart (100%) rename {pkgs => packages}/excerpter/test/updater_test.dart (100%) rename {pkgs => packages}/excerpter/test_data/example/dartdoc.dart (100%) rename {pkgs => packages}/excerpter/test_data/example/plaster.dart (100%) rename {pkgs => packages}/excerpter/test_data/example/simple_main.dart (100%) rename {pkgs => packages}/excerpter/test_data/example/simple_region.dart (100%) rename {pkgs => packages}/excerpter/test_data/example/transforms.dart (100%) rename {pkgs => packages}/excerpter/test_data/expected/dartdoc.md (100%) rename {pkgs => packages}/excerpter/test_data/expected/plaster.md (100%) rename {pkgs => packages}/excerpter/test_data/expected/simple.md (100%) rename {pkgs => packages}/excerpter/test_data/expected/transforms.md (100%) rename {pkgs => packages}/excerpter/test_data/src/dartdoc.md (100%) rename {pkgs => packages}/excerpter/test_data/src/plaster.md (100%) rename {pkgs => packages}/excerpter/test_data/src/simple.md (100%) rename {pkgs => packages}/excerpter/test_data/src/transforms.md (100%) diff --git a/README.md b/README.md index fe63dc87700..798d6326788 100644 --- a/README.md +++ b/README.md @@ -250,4 +250,4 @@ run `dart run dash_site refresh-excerpts`. To learn more about creating, editing, and using code excerpts, check out the [excerpt updater package documentation][]. -[excerpt updater package documentation]: https://github.com/dart-lang/site-shared/tree/main/pkgs/excerpter#readme +[excerpt updater package documentation]: https://github.com/flutter/website/tree/main/packages/excerpter#readme diff --git a/pkgs/analysis_defaults/analysis_options.yaml b/packages/analysis_defaults/analysis_options.yaml similarity index 100% rename from pkgs/analysis_defaults/analysis_options.yaml rename to packages/analysis_defaults/analysis_options.yaml diff --git a/pkgs/analysis_defaults/lib/analysis.yaml b/packages/analysis_defaults/lib/analysis.yaml similarity index 100% rename from pkgs/analysis_defaults/lib/analysis.yaml rename to packages/analysis_defaults/lib/analysis.yaml diff --git a/pkgs/analysis_defaults/pubspec.yaml b/packages/analysis_defaults/pubspec.yaml similarity index 100% rename from pkgs/analysis_defaults/pubspec.yaml rename to packages/analysis_defaults/pubspec.yaml diff --git a/pkgs/excerpter/LICENSE b/packages/excerpter/LICENSE similarity index 100% rename from pkgs/excerpter/LICENSE rename to packages/excerpter/LICENSE diff --git a/pkgs/excerpter/README.md b/packages/excerpter/README.md similarity index 100% rename from pkgs/excerpter/README.md rename to packages/excerpter/README.md diff --git a/pkgs/excerpter/analysis_options.yaml b/packages/excerpter/analysis_options.yaml similarity index 100% rename from pkgs/excerpter/analysis_options.yaml rename to packages/excerpter/analysis_options.yaml diff --git a/pkgs/excerpter/bin/excerpter.dart b/packages/excerpter/bin/excerpter.dart similarity index 100% rename from pkgs/excerpter/bin/excerpter.dart rename to packages/excerpter/bin/excerpter.dart diff --git a/pkgs/excerpter/lib/excerpter.dart b/packages/excerpter/lib/excerpter.dart similarity index 100% rename from pkgs/excerpter/lib/excerpter.dart rename to packages/excerpter/lib/excerpter.dart diff --git a/pkgs/excerpter/lib/src/extract.dart b/packages/excerpter/lib/src/extract.dart similarity index 100% rename from pkgs/excerpter/lib/src/extract.dart rename to packages/excerpter/lib/src/extract.dart diff --git a/pkgs/excerpter/lib/src/inject.dart b/packages/excerpter/lib/src/inject.dart similarity index 100% rename from pkgs/excerpter/lib/src/inject.dart rename to packages/excerpter/lib/src/inject.dart diff --git a/pkgs/excerpter/lib/src/transform.dart b/packages/excerpter/lib/src/transform.dart similarity index 100% rename from pkgs/excerpter/lib/src/transform.dart rename to packages/excerpter/lib/src/transform.dart diff --git a/pkgs/excerpter/lib/src/update.dart b/packages/excerpter/lib/src/update.dart similarity index 100% rename from pkgs/excerpter/lib/src/update.dart rename to packages/excerpter/lib/src/update.dart diff --git a/pkgs/excerpter/pubspec.yaml b/packages/excerpter/pubspec.yaml similarity index 100% rename from pkgs/excerpter/pubspec.yaml rename to packages/excerpter/pubspec.yaml diff --git a/pkgs/excerpter/test/cli_test.dart b/packages/excerpter/test/cli_test.dart similarity index 100% rename from pkgs/excerpter/test/cli_test.dart rename to packages/excerpter/test/cli_test.dart diff --git a/pkgs/excerpter/test/transform_test.dart b/packages/excerpter/test/transform_test.dart similarity index 100% rename from pkgs/excerpter/test/transform_test.dart rename to packages/excerpter/test/transform_test.dart diff --git a/pkgs/excerpter/test/updater_test.dart b/packages/excerpter/test/updater_test.dart similarity index 100% rename from pkgs/excerpter/test/updater_test.dart rename to packages/excerpter/test/updater_test.dart diff --git a/pkgs/excerpter/test_data/example/dartdoc.dart b/packages/excerpter/test_data/example/dartdoc.dart similarity index 100% rename from pkgs/excerpter/test_data/example/dartdoc.dart rename to packages/excerpter/test_data/example/dartdoc.dart diff --git a/pkgs/excerpter/test_data/example/plaster.dart b/packages/excerpter/test_data/example/plaster.dart similarity index 100% rename from pkgs/excerpter/test_data/example/plaster.dart rename to packages/excerpter/test_data/example/plaster.dart diff --git a/pkgs/excerpter/test_data/example/simple_main.dart b/packages/excerpter/test_data/example/simple_main.dart similarity index 100% rename from pkgs/excerpter/test_data/example/simple_main.dart rename to packages/excerpter/test_data/example/simple_main.dart diff --git a/pkgs/excerpter/test_data/example/simple_region.dart b/packages/excerpter/test_data/example/simple_region.dart similarity index 100% rename from pkgs/excerpter/test_data/example/simple_region.dart rename to packages/excerpter/test_data/example/simple_region.dart diff --git a/pkgs/excerpter/test_data/example/transforms.dart b/packages/excerpter/test_data/example/transforms.dart similarity index 100% rename from pkgs/excerpter/test_data/example/transforms.dart rename to packages/excerpter/test_data/example/transforms.dart diff --git a/pkgs/excerpter/test_data/expected/dartdoc.md b/packages/excerpter/test_data/expected/dartdoc.md similarity index 100% rename from pkgs/excerpter/test_data/expected/dartdoc.md rename to packages/excerpter/test_data/expected/dartdoc.md diff --git a/pkgs/excerpter/test_data/expected/plaster.md b/packages/excerpter/test_data/expected/plaster.md similarity index 100% rename from pkgs/excerpter/test_data/expected/plaster.md rename to packages/excerpter/test_data/expected/plaster.md diff --git a/pkgs/excerpter/test_data/expected/simple.md b/packages/excerpter/test_data/expected/simple.md similarity index 100% rename from pkgs/excerpter/test_data/expected/simple.md rename to packages/excerpter/test_data/expected/simple.md diff --git a/pkgs/excerpter/test_data/expected/transforms.md b/packages/excerpter/test_data/expected/transforms.md similarity index 100% rename from pkgs/excerpter/test_data/expected/transforms.md rename to packages/excerpter/test_data/expected/transforms.md diff --git a/pkgs/excerpter/test_data/src/dartdoc.md b/packages/excerpter/test_data/src/dartdoc.md similarity index 100% rename from pkgs/excerpter/test_data/src/dartdoc.md rename to packages/excerpter/test_data/src/dartdoc.md diff --git a/pkgs/excerpter/test_data/src/plaster.md b/packages/excerpter/test_data/src/plaster.md similarity index 100% rename from pkgs/excerpter/test_data/src/plaster.md rename to packages/excerpter/test_data/src/plaster.md diff --git a/pkgs/excerpter/test_data/src/simple.md b/packages/excerpter/test_data/src/simple.md similarity index 100% rename from pkgs/excerpter/test_data/src/simple.md rename to packages/excerpter/test_data/src/simple.md diff --git a/pkgs/excerpter/test_data/src/transforms.md b/packages/excerpter/test_data/src/transforms.md similarity index 100% rename from pkgs/excerpter/test_data/src/transforms.md rename to packages/excerpter/test_data/src/transforms.md diff --git a/pubspec.yaml b/pubspec.yaml index afcb53f3ac6..23981e17e37 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -5,7 +5,7 @@ environment: sdk: ^3.11.0 workspace: - - pkgs/analysis_defaults - - pkgs/excerpter + - packages/analysis_defaults + - packages/excerpter - site - tool/dash_site diff --git a/site/pubspec.yaml b/site/pubspec.yaml index 026183a31d8..e037c9f2bc3 100644 --- a/site/pubspec.yaml +++ b/site/pubspec.yaml @@ -29,7 +29,7 @@ dependencies: dev_dependencies: analysis_defaults: - path: ../pkgs/analysis_defaults + path: ../packages/analysis_defaults build_runner: ^2.11.0 build_web_compilers: ^4.4.9 jaspr_builder: ^0.22.3 diff --git a/src/content/contribute/docs/excerpts.md b/src/content/contribute/docs/excerpts.md index ae8258f1c4d..eff2df3c1d0 100644 --- a/src/content/contribute/docs/excerpts.md +++ b/src/content/contribute/docs/excerpts.md @@ -17,4 +17,4 @@ The source of code excerpts is in the root `/examples` directory. To learn how to use code excerpts, check out the [excerpter tool README][]. -[excerpter tool README]: https://github.com/dart-lang/site-shared/blob/main/pkgs/excerpter/README.md +[excerpter tool README]: https://github.com/flutter/website/blob/main/packages/excerpter/README.md diff --git a/tool/dash_site/pubspec.yaml b/tool/dash_site/pubspec.yaml index 3332728de35..8ebe90d7d2d 100644 --- a/tool/dash_site/pubspec.yaml +++ b/tool/dash_site/pubspec.yaml @@ -9,11 +9,11 @@ environment: dependencies: args: ^2.7.0 excerpter: - path: ../../pkgs/excerpter + path: ../../packages/excerpter io: ^1.0.5 linkcheck: ^3.1.0 path: ^1.9.1 dev_dependencies: analysis_defaults: - path: ../../pkgs/analysis_defaults + path: ../../packages/analysis_defaults From f308793606d38e38a485548d1bd924e2f6c2810b Mon Sep 17 00:00:00 2001 From: Parker Lougheed Date: Sat, 21 Mar 2026 19:54:39 +0800 Subject: [PATCH 3/4] Update remaining pkgs references --- tool/dash_site/lib/src/commands/analyze_dart.dart | 2 +- tool/dash_site/lib/src/commands/format_dart.dart | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tool/dash_site/lib/src/commands/analyze_dart.dart b/tool/dash_site/lib/src/commands/analyze_dart.dart index 660e0211e11..7be48fc2ddb 100644 --- a/tool/dash_site/lib/src/commands/analyze_dart.dart +++ b/tool/dash_site/lib/src/commands/analyze_dart.dart @@ -34,7 +34,7 @@ final class AnalyzeDartCommand extends Command { int analyzeDart({bool verboseLogging = false}) { final directoriesToAnalyze = [ 'examples', - 'pkgs', + 'packages', 'site', path.join('tool', 'dash_site'), ]; diff --git a/tool/dash_site/lib/src/commands/format_dart.dart b/tool/dash_site/lib/src/commands/format_dart.dart index 4403e994dec..40eef3972c7 100644 --- a/tool/dash_site/lib/src/commands/format_dart.dart +++ b/tool/dash_site/lib/src/commands/format_dart.dart @@ -5,6 +5,7 @@ import 'dart:io'; import 'package:args/command_runner.dart'; +import 'package:path/path.dart' as path; import '../utils.dart'; @@ -33,7 +34,12 @@ final class FormatDartCommand extends Command { } int formatDart({bool justCheck = false}) { - final directoriesToFormat = ['examples', 'pkgs', 'site', 'tool']; + final directoriesToFormat = [ + 'examples', + 'packages', + 'site', + path.join('tool', 'dash_site'), + ]; final dartFormatOutput = Process.runSync(Platform.resolvedExecutable, [ 'format', From de05088882a1546609d3421de60fa26fc635b3c4 Mon Sep 17 00:00:00 2001 From: Parker Lougheed Date: Sat, 21 Mar 2026 20:39:12 +0800 Subject: [PATCH 4/4] Apply suggestions from code review --- packages/excerpter/README.md | 3 --- packages/excerpter/test/updater_test.dart | 9 --------- 2 files changed, 12 deletions(-) diff --git a/packages/excerpter/README.md b/packages/excerpter/README.md index 47131da6f66..99082198e95 100644 --- a/packages/excerpter/README.md +++ b/packages/excerpter/README.md @@ -82,9 +82,6 @@ The `main-stub` region is discontinuous as it has a break in it. When this package is run to update excerpts, each break is replaced by a language-specific comment filled with a plaster marker (`...`). -For details concerning the processing of plasters, see the -[code_excerpt_updater][] documentation. - ## Injecting excerpts To inject content from docregions or entire files into Markdown files, diff --git a/packages/excerpter/test/updater_test.dart b/packages/excerpter/test/updater_test.dart index a21b07e6dcb..9e15c21b994 100644 --- a/packages/excerpter/test/updater_test.dart +++ b/packages/excerpter/test/updater_test.dart @@ -22,15 +22,6 @@ void _setup() { isTrue, ); }); - - test('includes default exclusions', () { - expect( - Updater(baseSourcePath: 'example', validTargetExtensions: {'.md'}) - .excludePaths - .any((excludePattern) => excludePattern.hasMatch('.dart_tool')), - isTrue, - ); - }); } void _defaultBehavior() {