Add a --charset flag and API option (#644)

The automatic @charset adding is useful in general, but there are
consistently cases where it trips up naïve downstream tools. This
option makes it easier for users to control when it occurs.

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 1.17.5
+## 1.18.0
 
 * Avoid recursively listing directories when finding the canonical name of a
   file on case-insensitive filesystems.
@@ -7,8 +7,19 @@
 
 * Don't claim that "package:" URLs aren't supported when they actually are.
 
+### Command-Line Interface
+
+* Add a `--no-charset` flag. If this flag is set, Sass will never emit a
+  `@charset` declaration or a byte-order mark, even if the CSS file contains
+  non-ASCII characters.
+
 ### Dart API
 
+* Add a `charset` option to `compile()`, `compileString()`, `compileAsync()` and
+  `compileStringAsync()`. If this option is set to `false`, Sass will never emit
+  a `@charset` declaration or a byte-order mark, even if the CSS file contains
+  non-ASCII characters.
+
 * Explicitly require that importers' `canonicalize()` methods be able to take
   paths relative to their outputs as valid inputs. This isn't considered a
   breaking change because the importer infrastructure already required this in

lib/sass.dart

@@ -70,6 +70,13 @@ export 'src/visitor/serialize.dart' show OutputStyle;
 ///
 /// [`source_maps`]: https://pub.dartlang.org/packages/source_maps
 ///
+/// If [charset] is `true`, this will include a `@charset` declaration or a
+/// UTF-8 [byte-order mark][] if the stylesheet contains any non-ASCII
+/// characters. Otherwise, it will never include a `@charset` declaration or a
+/// byte-order mark.
+///
+/// [byte-order mark]: https://en.wikipedia.org/wiki/Byte_order_mark#UTF-8
+///
 /// This parameter is meant to be used as an out parameter, so that users who
 /// want access to the source map can get it. For example:
 ///
@@ -87,7 +94,8 @@ String compile(String path,
     SyncPackageResolver packageResolver,
     Iterable<Callable> functions,
     OutputStyle style,
-    void sourceMap(SingleMapping map)}) {
+    void sourceMap(SingleMapping map),
+    bool charset = true}) {
   logger ??= Logger.stderr(color: color);
   var result = c.compile(path,
       logger: logger,
@@ -97,7 +105,8 @@ String compile(String path,
           packageResolver: packageResolver),
       functions: functions,
       style: style,
-      sourceMap: sourceMap != null);
+      sourceMap: sourceMap != null,
+      charset: charset);
   if (sourceMap != null) sourceMap(result.sourceMap);
   return result.css;
 }
@@ -149,6 +158,13 @@ String compile(String path,
 ///
 /// [`source_maps`]: https://pub.dartlang.org/packages/source_maps
 ///
+/// If [charset] is `true`, this will include a `@charset` declaration or a
+/// UTF-8 [byte-order mark][] if the stylesheet contains any non-ASCII
+/// characters. Otherwise, it will never include a `@charset` declaration or a
+/// byte-order mark.
+///
+/// [byte-order mark]: https://en.wikipedia.org/wiki/Byte_order_mark#UTF-8
+///
 /// This parameter is meant to be used as an out parameter, so that users who
 /// want access to the source map can get it. For example:
 ///
@@ -170,6 +186,7 @@ String compileString(String source,
     Importer importer,
     url,
     void sourceMap(SingleMapping map),
+    bool charset = true,
     @Deprecated("Use syntax instead.") bool indented = false}) {
   logger ??= Logger.stderr(color: color);
   var result = c.compileString(source,
@@ -183,7 +200,8 @@ String compileString(String source,
       style: style,
       importer: importer,
       url: url,
-      sourceMap: sourceMap != null);
+      sourceMap: sourceMap != null,
+      charset: charset);
   if (sourceMap != null) sourceMap(result.sourceMap);
   return result.css;
 }
@@ -233,6 +251,7 @@ Future<String> compileStringAsync(String source,
     AsyncImporter importer,
     url,
     void sourceMap(SingleMapping map),
+    bool charset = true,
     @Deprecated("Use syntax instead.") bool indented = false}) async {
   logger ??= Logger.stderr(color: color);
   var result = await c.compileStringAsync(source,
@@ -246,7 +265,8 @@ Future<String> compileStringAsync(String source,
       style: style,
       importer: importer,
       url: url,
-      sourceMap: sourceMap != null);
+      sourceMap: sourceMap != null,
+      charset: charset);
   if (sourceMap != null) sourceMap(result.sourceMap);
   return result.css;
 }

lib/src/async_compile.dart

@@ -36,7 +36,8 @@ Future<CompileResult> compileAsync(String path,
     bool useSpaces = true,
     int indentWidth,
     LineFeed lineFeed,
-    bool sourceMap = false}) async {
+    bool sourceMap = false,
+    bool charset = true}) async {
   // If the syntax is different than the importer would default to, we have to
   // parse the file manually and we can't store it in the cache.
   Stylesheet stylesheet;
@@ -62,7 +63,8 @@ Future<CompileResult> compileAsync(String path,
       useSpaces,
       indentWidth,
       lineFeed,
-      sourceMap);
+      sourceMap,
+      charset);
 }
 
 /// Like [compileStringAsync] in `lib/sass.dart`, but provides more options to
@@ -84,7 +86,8 @@ Future<CompileResult> compileStringAsync(String source,
     int indentWidth,
     LineFeed lineFeed,
     url,
-    bool sourceMap = false}) async {
+    bool sourceMap = false,
+    bool charset = true}) async {
   var stylesheet =
       Stylesheet.parse(source, syntax ?? Syntax.scss, url: url, logger: logger);
 
@@ -99,7 +102,8 @@ Future<CompileResult> compileStringAsync(String source,
       useSpaces,
       indentWidth,
       lineFeed,
-      sourceMap);
+      sourceMap,
+      charset);
 }
 
 /// Compiles [stylesheet] and returns its result.
@@ -116,7 +120,8 @@ Future<CompileResult> _compileStylesheet(
     bool useSpaces,
     int indentWidth,
     LineFeed lineFeed,
-    bool sourceMap) async {
+    bool sourceMap,
+    bool charset) async {
   var evaluateResult = await evaluateAsync(stylesheet,
       importCache: importCache,
       nodeImporter: nodeImporter,
@@ -130,7 +135,8 @@ Future<CompileResult> _compileStylesheet(
       useSpaces: useSpaces,
       indentWidth: indentWidth,
       lineFeed: lineFeed,
-      sourceMap: sourceMap);
+      sourceMap: sourceMap,
+      charset: charset);
 
   if (serializeResult.sourceMap != null && importCache != null) {
     // TODO(nweiz): Don't explicitly use a type parameter when dart-lang/sdk#25490

lib/src/compile.dart

@@ -5,7 +5,7 @@
 // DO NOT EDIT. This file was generated from async_compile.dart.
 // See tool/synchronize.dart for details.
 //
-// Checksum: 2bb00947655b3add16335253802a82188d730595
+// Checksum: ea78ec4431055c1d222e52f4ea54a9659c4df11f
 //
 // ignore_for_file: unused_import
 
@@ -45,7 +45,8 @@ CompileResult compile(String path,
     bool useSpaces = true,
     int indentWidth,
     LineFeed lineFeed,
-    bool sourceMap = false}) {
+    bool sourceMap = false,
+    bool charset = true}) {
   // If the syntax is different than the importer would default to, we have to
   // parse the file manually and we can't store it in the cache.
   Stylesheet stylesheet;
@@ -71,7 +72,8 @@ CompileResult compile(String path,
       useSpaces,
       indentWidth,
       lineFeed,
-      sourceMap);
+      sourceMap,
+      charset);
 }
 
 /// Like [compileString] in `lib/sass.dart`, but provides more options to
@@ -93,7 +95,8 @@ CompileResult compileString(String source,
     int indentWidth,
     LineFeed lineFeed,
     url,
-    bool sourceMap = false}) {
+    bool sourceMap = false,
+    bool charset = true}) {
   var stylesheet =
       Stylesheet.parse(source, syntax ?? Syntax.scss, url: url, logger: logger);
 
@@ -108,7 +111,8 @@ CompileResult compileString(String source,
       useSpaces,
       indentWidth,
       lineFeed,
-      sourceMap);
+      sourceMap,
+      charset);
 }
 
 /// Compiles [stylesheet] and returns its result.
@@ -125,7 +129,8 @@ CompileResult _compileStylesheet(
     bool useSpaces,
     int indentWidth,
     LineFeed lineFeed,
-    bool sourceMap) {
+    bool sourceMap,
+    bool charset) {
   var evaluateResult = evaluate(stylesheet,
       importCache: importCache,
       nodeImporter: nodeImporter,
@@ -139,7 +144,8 @@ CompileResult _compileStylesheet(
       useSpaces: useSpaces,
       indentWidth: indentWidth,
       lineFeed: lineFeed,
-      sourceMap: sourceMap);
+      sourceMap: sourceMap,
+      charset: charset);
 
   if (serializeResult.sourceMap != null && importCache != null) {
     // TODO(nweiz): Don't explicitly use a type parameter when dart-lang/sdk#25490

lib/src/executable/compile_stylesheet.dart

@@ -67,13 +67,15 @@ Future compileStylesheet(ExecutableOptions options, StylesheetGraph graph,
             importCache: importCache,
             importer: FilesystemImporter('.'),
             style: options.style,
-            sourceMap: options.emitSourceMap)
+            sourceMap: options.emitSourceMap,
+            charset: options.charset)
         : await compileAsync(source,
             syntax: syntax,
             logger: options.logger,
             importCache: importCache,
             style: options.style,
-            sourceMap: options.emitSourceMap);
+            sourceMap: options.emitSourceMap,
+            charset: options.charset);
   } else {
     result = source == null
         ? compileString(await readStdin(),
@@ -82,13 +84,15 @@ Future compileStylesheet(ExecutableOptions options, StylesheetGraph graph,
             importCache: graph.importCache,
             importer: FilesystemImporter('.'),
             style: options.style,
-            sourceMap: options.emitSourceMap)
+            sourceMap: options.emitSourceMap,
+            charset: options.charset)
         : compile(source,
             syntax: syntax,
             logger: options.logger,
             importCache: graph.importCache,
             style: options.style,
-            sourceMap: options.emitSourceMap);
+            sourceMap: options.emitSourceMap,
+            charset: options.charset);
   }
 
   var css = result.css;

lib/src/executable/options.dart

@@ -55,6 +55,9 @@ class ExecutableOptions {
           help: 'Output style.',
           allowed: ['expanded', 'compressed'],
           defaultsTo: 'expanded')
+      ..addFlag('charset',
+          help: 'Emit a @charset or BOM for CSS with non-ASCII characters.',
+          defaultsTo: true)
       ..addFlag('update',
           help: 'Only compile out-of-date stylesheets.', negatable: false);
 
@@ -171,6 +174,10 @@ class ExecutableOptions {
       ? OutputStyle.compressed
       : OutputStyle.expanded;
 
+  /// Whether to include a `@charset` declaration or a BOM if the stylesheet
+  /// contains any non-ASCII characters.
+  bool get charset => _options['charset'] as bool;
+
   /// The set of paths Sass in which should look for imported files.
   List<String> get loadPaths => _options['load-path'] as List<String>;
 

lib/src/visitor/serialize.dart

@@ -39,13 +39,17 @@ import 'interface/value.dart';
 ///
 /// If [sourceMap] is `true`, the returned [SerializeResult] will contain a
 /// source map indicating how the original Sass files map to the compiled CSS.
+///
+/// If [charset] is `true`, this will include a `@charset` declaration or a BOM
+/// if the stylesheet contains any non-ASCII characters.
 SerializeResult serialize(CssNode node,
     {OutputStyle style,
     bool inspect = false,
     bool useSpaces = true,
     int indentWidth,
     LineFeed lineFeed,
-    bool sourceMap = false}) {
+    bool sourceMap = false,
+    bool charset = true}) {
   indentWidth ??= 2;
   var visitor = _SerializeVisitor(
       style: style,
@@ -57,7 +61,7 @@ SerializeResult serialize(CssNode node,
   node.accept(visitor);
   var css = visitor._buffer.toString();
   String prefix;
-  if (css.codeUnits.any((codeUnit) => codeUnit > 0x7F)) {
+  if (charset && css.codeUnits.any((codeUnit) => codeUnit > 0x7F)) {
     if (style == OutputStyle.compressed) {
       prefix = '\uFEFF';
     } else {

pubspec.yaml

@@ -1,5 +1,5 @@
 name: sass
-version: 1.17.5-dev
+version: 1.18.0
 description: A Sass implementation in Dart.
 author: Dart Team <misc@dartlang.org>
 homepage: https://github.com/sass/dart-sass

test/cli/shared.dart

@@ -3,6 +3,7 @@
 // https://opensource.org/licenses/MIT.
 
 import 'dart:async';
+import 'dart:convert';
 import 'dart:io';
 
 import 'package:path/path.dart' as p;
@@ -389,4 +390,90 @@ void sharedTests(
       await sass.shouldExit(0);
     });
   });
+
+  group("with --charset", () {
+    test("doesn't emit @charset for a pure-ASCII stylesheet", () async {
+      await d.file("test.scss", "a {b: c}").create();
+
+      var sass = await runSass(["test.scss"]);
+      expect(
+          sass.stdout,
+          emitsInOrder([
+            "a {",
+            "  b: c;",
+            "}",
+          ]));
+      await sass.shouldExit(0);
+    });
+
+    test("emits @charset with expanded output", () async {
+      await d.file("test.scss", "a {b: 👭}").create();
+
+      var sass = await runSass(["test.scss"]);
+      expect(
+          sass.stdout,
+          emitsInOrder([
+            "@charset \"UTF-8\";",
+            "a {",
+            "  b: 👭;",
+            "}",
+          ]));
+      await sass.shouldExit(0);
+    });
+
+    test("emits a BOM with compressed output", () async {
+      await d.file("test.scss", "a {b: 👭}").create();
+
+      var sass = await runSass(
+          ["--no-source-map", "--style=compressed", "test.scss", "test.css"]);
+      await sass.shouldExit(0);
+
+      // We can't verify this as a string because `dart:io` automatically trims
+      // the BOM.
+      var bomBytes = utf8.encode("\uFEFF");
+      expect(
+          File(p.join(d.sandbox, "test.css"))
+              .readAsBytesSync()
+              .sublist(0, bomBytes.length),
+          equals(bomBytes));
+    });
+  });
+
+  group("with --no-charset", () {
+    test("doesn't emit @charset with expanded output", () async {
+      await d.file("test.scss", "a {b: 👭}").create();
+
+      var sass = await runSass(["--no-charset", "test.scss"]);
+      expect(
+          sass.stdout,
+          emitsInOrder([
+            "a {",
+            "  b: 👭;",
+            "}",
+          ]));
+      await sass.shouldExit(0);
+    });
+
+    test("doesn't emit a BOM with compressed output", () async {
+      await d.file("test.scss", "a {b: 👭}").create();
+
+      var sass = await runSass([
+        "--no-charset",
+        "--no-source-map",
+        "--style=compressed",
+        "test.scss",
+        "test.css"
+      ]);
+      await sass.shouldExit(0);
+
+      // We can't verify this as a string because `dart:io` automatically trims
+      // the BOM.
+      var bomBytes = utf8.encode("\uFEFF");
+      expect(
+          File(p.join(d.sandbox, "test.css"))
+              .readAsBytesSync()
+              .sublist(0, bomBytes.length),
+          isNot(equals(bomBytes)));
+    });
+  });
 }

test/dart_api_test.dart

@@ -185,4 +185,44 @@ main() {
       expect(css, equals("a {\n  b: from-importer;\n}"));
     });
   });
+
+  group("charset", () {
+    group("= true", () {
+      test("doesn't emit @charset for a pure-ASCII stylesheet", () {
+        expect(compileString("a {b: c}"), equals("""
+a {
+  b: c;
+}"""));
+      });
+
+      test("emits @charset with expanded output", () async {
+        expect(compileString("a {b: 👭}"), equals("""
+@charset "UTF-8";
+a {
+  b: 👭;
+}"""));
+      });
+
+      test("emits a BOM with compressed output", () async {
+        expect(compileString("a {b: 👭}", style: OutputStyle.compressed),
+            equals("\u{FEFF}a{b:👭}"));
+      });
+    });
+
+    group("= false", () {
+      test("doesn't emit @charset with expanded output", () async {
+        expect(compileString("a {b: 👭}", charset: false), equals("""
+a {
+  b: 👭;
+}"""));
+      });
+
+      test("emits a BOM with compressed output", () async {
+        expect(
+            compileString("a {b: 👭}",
+                charset: false, style: OutputStyle.compressed),
+            equals("a{b:👭}"));
+      });
+    });
+  });
 }