Merge pull request #2642 from rust-lang/tshepang/sembr

sembr stabilization_guide.md and improve it a bit

Author: tshepang

ci/sembr/src/main.rs

@@ -22,13 +22,14 @@ struct Cli {
     show_diff: bool,
 }
 
-static REGEX_IGNORE: LazyLock<Regex> =
-    LazyLock::new(|| Regex::new(r"^\s*(\d\.|\-|\*)\s+").unwrap());
 static REGEX_IGNORE_END: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(\.|\?|;|!)$").unwrap());
 static REGEX_IGNORE_LINK_TARGETS: LazyLock<Regex> =
     LazyLock::new(|| Regex::new(r"^\[.+\]: ").unwrap());
 static REGEX_SPLIT: LazyLock<Regex> =
-    LazyLock::new(|| Regex::new(r"([^\.]\.|[^r]\?|;|!)\s+").unwrap());
+    LazyLock::new(|| Regex::new(r"([^\.\d\-\*]\.|[^r]\?|;|!)\s").unwrap());
+// list elements, numbered (1.) or not  (- and *)
+static REGEX_LIST_ENTRY: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"^\s*(\d\.|\-|\*)\s+").unwrap());
 
 fn main() -> Result<()> {
     let cli = Cli::parse();
@@ -99,7 +100,6 @@ fn ignore(line: &str, in_code_block: bool) -> bool {
         || line.trim_start().starts_with('>')
         || line.starts_with('#')
         || line.trim().is_empty()
-        || REGEX_IGNORE.is_match(line)
         || REGEX_IGNORE_LINK_TARGETS.is_match(line)
 }
 
@@ -120,11 +120,19 @@ fn comply(content: &str) -> String {
             continue;
         }
         if REGEX_SPLIT.is_match(&line) {
-            let indent = line.find(|ch: char| !ch.is_whitespace()).unwrap();
-            let new_lines: Vec<_> = line
-                .split_inclusive(&*REGEX_SPLIT)
-                .map(|portion| format!("{:indent$}{}", "", portion.trim()))
+            let indent = if let Some(regex_match) = REGEX_LIST_ENTRY.find(&line) {
+                regex_match.len()
+            } else {
+                line.find(|ch: char| !ch.is_whitespace()).unwrap()
+            };
+            let mut newly_split_lines = line.split_inclusive(&*REGEX_SPLIT);
+            let first = newly_split_lines.next().unwrap().trim_end().to_owned();
+            let mut remaining: Vec<_> = newly_split_lines
+                .map(|portion| format!("{:indent$}{}", "", portion.trim_end()))
                 .collect();
+            let mut new_lines = Vec::new();
+            new_lines.push(first);
+            new_lines.append(&mut remaining);
             new_content.splice(new_n..=new_n, new_lines.clone());
             new_n += new_lines.len() - 1;
         }
@@ -182,42 +190,47 @@ fn lengthen_lines(content: &str, limit: usize) -> String {
 
 #[test]
 fn test_sembr() {
-    let original = "\
+    let original = "
 # some. heading
-must! be; split?  and.   normalizes space
-1. ignore numbered
+must! be; split?
+1. ignore a dot after number. but no further
 ignore | tables
 ignore e.g. and
 ignore i.e. and
 ignore E.g. too
-- ignore. list
-* ignore. list
+- list. entry
+ * list. entry
 ```
 some code. block
 ```
 sentence with *italics* should not be ignored. truly.
 git log main.. compiler
+ foo.   bar.  baz
 ";
-    let expected = "\
+    let expected = "
 # some. heading
 must!
 be;
 split?
-and.
-normalizes space
-1. ignore numbered
+1. ignore a dot after number.
+   but no further
 ignore | tables
 ignore e.g. and
 ignore i.e. and
 ignore E.g. too
-- ignore. list
-* ignore. list
+- list.
+  entry
+ * list.
+   entry
 ```
 some code. block
 ```
 sentence with *italics* should not be ignored.
 truly.
 git log main.. compiler
+ foo.
+   bar.
+  baz
 ";
     assert_eq!(expected, comply(original));
 }
@@ -263,13 +276,13 @@ fn test_prettify_ignore_link_targets() {
 
 #[test]
 fn test_sembr_then_prettify() {
-    let original = "\
+    let original = "
 hi there. do
 not split
 short sentences.
 hi again.
 ";
-    let expected = "\
+    let expected = "
 hi there.
 do
 not split
@@ -278,15 +291,15 @@ hi again.
 ";
     let processed = comply(original);
     assert_eq!(expected, processed);
-    let expected = "\
+    let expected = "
 hi there.
 do not split
 short sentences.
 hi again.
 ";
     let processed = lengthen_lines(&processed, 50);
     assert_eq!(expected, processed);
-    let expected = "\
+    let expected = "
 hi there.
 do not split short sentences.
 hi again.
@@ -297,12 +310,12 @@ hi again.
 
 #[test]
 fn test_sembr_question_mark() {
-    let original = "\
+    let original = "
 o? whatever
 r? @reviewer
  r? @reviewer
 ";
-    let expected = "\
+    let expected = "
 o?
 whatever
 r? @reviewer

src/stabilization_guide.md

@@ -1,10 +1,12 @@
 # Request for stabilization
 
-**NOTE**: This page is about stabilizing *language* features. For stabilizing *library* features, see [Stabilizing a library feature].
+**NOTE**: This page is about stabilizing *language* features.
+For stabilizing *library* features, see [Stabilizing a library feature].
 
 [Stabilizing a library feature]: ./stability.md#stabilizing-a-library-feature
 
-Once an unstable feature has been well-tested with no outstanding concerns, anyone may push for its stabilization, though involving the people who have worked on it is prudent. Follow these steps:
+Once an unstable feature has been well-tested with no outstanding concerns, anyone may push for its stabilization, though involving the people who have worked on it is prudent.
+Follow these steps:
 
 ## Write an RFC, if needed
 
@@ -16,16 +18,23 @@ If the feature was part of a [lang experiment], the lang team generally will wan
 
 <a id="updating-documentation"></a>
 
-The feature might be documented in the [`Unstable Book`], located at [`src/doc/unstable-book`]. Remove the page for the feature gate if it exists. Integrate any useful parts of that documentation in other places.
+The feature might be documented in the [`Unstable Book`], located at [`src/doc/unstable-book`].
+Remove the page for the feature gate if it exists.
+Integrate any useful parts of that documentation in other places.
 
 Places that may need updated documentation include:
 
 - [The Reference]: This must be updated, in full detail, and a member of the lang-docs team must review and approve the PR before the stabilization can be merged.
-- [The Book]: This is updated as needed. If you're not sure, please open an issue on this repository and it can be discussed.
-- Standard library documentation: This is updated as needed. Language features often don't need this, but if it's a feature that changes how idiomatic examples are written, such as when `?` was added to the language, updating these in the library documentation is important. Review also the keyword documentation and ABI documentation in the standard library, as these sometimes needs updates for language changes.
+- [The Book]: This is updated as needed.
+  If you're not sure, please open an issue on this repository and it can be discussed.
+- Standard library documentation: This is updated as needed.
+  Language features often don't need this, but if it's a feature that changes how idiomatic examples are written, such as when `?` was added to the language, updating these in the library documentation is important.
+  Review also the keyword documentation and ABI documentation in the standard library, as these sometimes need updates for language changes.
 - [Rust by Example]: This is updated as needed.
 
-Prepare PRs to update documentation involving this new feature for the repositories mentioned above. Maintainers of these repositories will keep these PRs open until the whole stabilization process has completed. Meanwhile, we can proceed to the next step.
+Prepare PRs to update documentation involving this new feature for the repositories mentioned above.
+Maintainers of these repositories will keep these PRs open until the whole stabilization process has completed.
+Meanwhile, we can proceed to the next step.
 
 ## Write a stabilization report
 
@@ -34,7 +43,8 @@ Author a stabilization report using the [template found in this repository][srt]
 The stabilization reports summarizes:
 
 - The main design decisions and deviations since the RFC was accepted, including both decisions that were FCP'd or otherwise accepted by the language team as well as those being presented to the lang team for the first time.
-    - Often, the final stabilized language feature has significant design deviations from the original RFC. That's OK, but these deviations must be highlighted and explained carefully.
+    - Often, the final stabilized language feature has significant design deviations from the original RFC.
+      That's OK, but these deviations must be highlighted and explained carefully.
 - The work that has been done since the RFC was accepted, acknowledging the main contributors that helped drive the language feature forward.
 
 The [*Stabilization Template*][srt] includes a series of questions that aim to surface connections between this feature and lang's subteams (e.g. types, opsem, lang-docs, etc.) and to identify items that are commonly overlooked.
@@ -51,14 +61,19 @@ Before the stabilization will be considered by the lang team, there must be a co
 
 ### Updating the feature-gate listing
 
-There is a central listing of unstable feature-gates in [`compiler/rustc_feature/src/unstable.rs`]. Search for the `declare_features!`  macro. There should be an entry for the feature you are aiming to stabilize, something like (this example is taken from [rust-lang/rust#32409]:
+There is a central listing of unstable feature-gates in [`compiler/rustc_feature/src/unstable.rs`].
+Search for the `declare_features!`  macro.
+There should be an entry for the feature you are aiming to stabilize,
+something like the following (taken from [rust-lang/rust#32409]:
 
 ```rust,ignore
 // pub(restricted) visibilities (RFC 1422)
 (unstable, pub_restricted, "CURRENT_RUSTC_VERSION", Some(32409)),
 ```
 
-The above line should be moved to [`compiler/rustc_feature/src/accepted.rs`]. Entries in the `declare_features!` call are sorted, so find the correct place. When it is done, it should look like:
+The above line should be moved to [`compiler/rustc_feature/src/accepted.rs`].
+Entries in the `declare_features!` call are sorted, so find the correct place.
+When it is done, it should look like:
 
 ```rust,ignore
 // pub(restricted) visibilities (RFC 1422)
@@ -70,27 +85,36 @@ The above line should be moved to [`compiler/rustc_feature/src/accepted.rs`]. En
 
 ### Removing existing uses of the feature-gate
 
-Next, search for the feature string (in this case, `pub_restricted`) in the codebase to find where it appears. Change uses of `#![feature(XXX)]` from the `std` and any rustc crates (this includes test folders under `library/` and `compiler/` but not the toplevel `tests/` one) to be `#![cfg_attr(bootstrap, feature(XXX))]`. This includes the feature-gate only for stage0, which is built using the current beta (this is needed because the feature is still unstable in the current beta).
+Next, search for the feature string (in this case, `pub_restricted`) in the codebase to find where it appears.
+Change uses of `#![feature(XXX)]` from the `std` and any rustc crates
+(which includes test folders under `library/` and `compiler/` but not the toplevel `tests/` one)
+to be `#![cfg_attr(bootstrap, feature(XXX))]`.
+This includes the feature-gate only for stage0, which is built using the current beta (this is needed because the feature is still unstable in the current beta).
 
 Also, remove those strings from any tests (e.g. under `tests/`). If there are tests specifically targeting the feature-gate (i.e., testing that the feature-gate is required to use the feature, but nothing else), simply remove the test.
 
 ### Do not require the feature-gate to use the feature
 
-Most importantly, remove the code which flags an error if the feature-gate is not present (since the feature is now considered stable). If the feature can be detected because it employs some new syntax, then a common place for that code to be is in `compiler/rustc_ast_passes/src/feature_gate.rs`. For example, you might see code like this:
+Most importantly, remove the code which flags an error if the feature-gate is not present (since the feature is now considered stable).
+If the feature can be detected because it employs some new syntax, then a common place for that code to be is in `compiler/rustc_ast_passes/src/feature_gate.rs`.
+For example, you might see code like this:
 
 ```rust,ignore
 gate_all!(pub_restricted, "`pub(restricted)` syntax is experimental");
 ```
 
-The `gate_all!` macro reports an error if the `pub_restricted` feature is not enabled. It is not needed now that `pub(restricted)` is stable.
+The `gate_all!` macro reports an error if the `pub_restricted` feature is not enabled.
+It is not needed now that `pub(restricted)` is stable.
 
 For more subtle features, you may find code like this:
 
 ```rust,ignore
 if self.tcx.features().async_fn_in_dyn_trait() { /* XXX */ }
 ```
 
-This `pub_restricted` field (named after the feature) would ordinarily be false if the feature flag is not present and true if it is. So transform the code to assume that the field is true. In this case, that would mean removing the `if` and leaving just the `/* XXX */`.
+This `pub_restricted` field (named after the feature) would ordinarily be false if the feature flag is not present and true if it is.
+So, transform the code to assume that the field is true.
+In this case, that would mean removing the `if` and leaving just the `/* XXX */`.
 
 ```rust,ignore
 if self.tcx.sess.features.borrow().pub_restricted { /* XXX */ }
@@ -126,7 +150,8 @@ When opening the stabilization PR, CC the lang team and its advisors (`@rust-lan
 - `@rust-lang/libs-api`, for changes to the standard library API or its guarantees.
 - `@rust-lang/lang-docs`, for questions about how this should be documented in the Reference.
 
-After the stabilization PR is opened with the stabilization report, wait a bit for any immediate comments. When such comments "simmer down" and you feel the PR is ready for consideration by the lang team, [nominate the PR](https://lang-team.rust-lang.org/how_to/nominate.html) to get it on the agenda for consideration in an upcoming lang meeting.
+After the stabilization PR is opened with the stabilization report, wait a bit for any immediate comments.
+When such comments "simmer down" and you feel the PR is ready for consideration by the lang team, [nominate the PR](https://lang-team.rust-lang.org/how_to/nominate.html) to get it on the agenda for consideration in an upcoming lang meeting.
 
 If you are not a `rust-lang` organization member, you can ask your assigned reviewer to CC the relevant teams on your behalf.
 
@@ -138,7 +163,8 @@ After the lang team and other relevant teams review the stabilization, and after
 @rfcbot fcp merge
 ```
 
-Once enough team members have reviewed, the PR will move into a "final comment period" (FCP). If no new concerns are raised, this period will complete and the PR can be merged after implementation review in the usual way.
+Once enough team members have reviewed, the PR will move into a "final comment period" (FCP).
+If no new concerns are raised, this period will complete and the PR can be merged after implementation review in the usual way.
 
 ## Reviewing and merging stabilizations
 
@@ -151,4 +177,5 @@ On a stabilization, before giving it the `r+`, ensure that the PR:
 - Has sufficient tests to convincingly demonstrate these things.
 - Is accompanied by a PR to the Reference than has been reviewed and approved by a member of lang-docs.
 
-In particular, when reviewing the PR, keep an eye out for any user-visible details that the lang team failed to consider and specify. If you find one, describe it and nominate the PR for the lang team.
+In particular, when reviewing the PR, keep an eye out for any user-visible details that the lang team failed to consider and specify.
+If you find one, describe it and nominate the PR for the lang team.