Pass lists/attrsets to bash as (associative) arrays

Author: edolstra

src/libexpr/primops.cc

@@ -713,7 +713,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
 
         Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
-        drv.env["out"] = outPath;
+        if (!jsonObject) drv.env["out"] = outPath;
         drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash);
     }
 
@@ -724,7 +724,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
            an empty value.  This ensures that changes in the set of
            output names do get reflected in the hash. */
         for (auto & i : outputs) {
-            drv.env[i] = "";
+            if (!jsonObject) drv.env[i] = "";
             drv.outputs[i] = DerivationOutput("", "", "");
         }
 
@@ -735,7 +735,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         for (auto & i : drv.outputs)
             if (i.second.path == "") {
                 Path outPath = state.store->makeOutputPath(i.first, h, drvName);
-                drv.env[i.first] = outPath;
+                if (!jsonObject) drv.env[i.first] = outPath;
                 i.second.path = outPath;
             }
     }

src/libstore/build.cc

@@ -18,6 +18,7 @@
 #include <thread>
 #include <future>
 #include <chrono>
+#include <regex>
 
 #include <limits.h>
 #include <sys/time.h>
@@ -55,6 +56,8 @@
 #include <sys/statvfs.h>
 #endif
 
+#include <nlohmann/json.hpp>
+
 
 namespace nix {
 
@@ -2286,12 +2289,99 @@ void DerivationGoal::initEnv()
 }
 
 
+static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
+
+
 void DerivationGoal::writeStructuredAttrs()
 {
-    auto json = drv->env.find("__json");
-    if (json == drv->env.end()) return;
+    auto jsonAttr = drv->env.find("__json");
+    if (jsonAttr == drv->env.end()) return;
+
+    try {
+
+        auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites);
+
+        auto json = nlohmann::json::parse(jsonStr);
+
+        /* Add an "outputs" object containing the output paths. */
+        nlohmann::json outputs;
+        for (auto & i : drv->outputs)
+            outputs[i.first] = rewriteStrings(i.second.path, inputRewrites);
+        json["outputs"] = outputs;
+
+        writeFile(tmpDir + "/.attrs.json", json.dump());
+
+        /* As a convenience to bash scripts, write a shell file that
+           maps all attributes that are representable in bash -
+           namely, strings, integers, nulls, Booleans, and arrays and
+           objects consisting entirely of those values. (So nested
+           arrays or objects are not supported.) */
+
+        auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> {
+            if (value.is_string())
+                return shellEscape(value);
+
+            if (value.is_number()) {
+                auto f = value.get<float>();
+                if (std::ceil(f) == f)
+                    return std::to_string(value.get<int>());
+            }
+
+            if (value.is_null())
+                return "''";
+
+            if (value.is_boolean())
+                return value.get<bool>() ? "1" : "";
+
+            return {};
+        };
+
+        std::string jsonSh;
 
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json->second, inputRewrites));
+        for (auto i = json.begin(); i != json.end(); ++i) {
+
+            if (!std::regex_match(i.key(), shVarName)) continue;
+
+            auto & value = i.value();
+
+            auto s = handleSimpleType(value);
+            if (s)
+                jsonSh += fmt("declare %s=%s\n", i.key(), *s);
+
+            else if (value.is_array()) {
+                std::string s2;
+                bool good = true;
+
+                for (auto i = value.begin(); i != value.end(); ++i) {
+                    auto s3 = handleSimpleType(i.value());
+                    if (!s3) { good = false; break; }
+                    s2 += *s3; s2 += ' ';
+                }
+
+                if (good)
+                    jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
+            }
+
+            else if (value.is_object()) {
+                std::string s2;
+                bool good = true;
+
+                for (auto i = value.begin(); i != value.end(); ++i) {
+                    auto s3 = handleSimpleType(i.value());
+                    if (!s3) { good = false; break; }
+                    s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
+                }
+
+                if (good)
+                    jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
+            }
+        }
+
+        writeFile(tmpDir + "/.attrs.sh", jsonSh);
+
+    } catch (std::exception & e) {
+        throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
+    }
 }
 
 

src/libutil/util.cc

@@ -1142,6 +1142,16 @@ std::string toLower(const std::string & s)
 }
 
 
+std::string shellEscape(const std::string & s)
+{
+    std::string r = "'";
+    for (auto & i : s)
+        if (i == '\'') r += "'\\''"; else r += i;
+    r += '\'';
+    return r;
+}
+
+
 void ignoreException()
 {
     try {

src/libutil/util.hh

@@ -352,10 +352,8 @@ bool hasSuffix(const string & s, const string & suffix);
 std::string toLower(const std::string & s);
 
 
-/* Escape a string that contains octal-encoded escape codes such as
-   used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to
-   "foo bar"). */
-string decodeOctalEscaped(const string & s);
+/* Escape a string as a shell word. */
+std::string shellEscape(const std::string & s);
 
 
 /* Exception handling in destructors: print an error message, then

src/nix-build/nix-build.cc

@@ -196,10 +196,6 @@ void mainWrapped(int argc, char * * argv)
             interactive = false;
             auto execArgs = "";
 
-            auto shellEscape = [](const string & s) {
-                return "'" + std::regex_replace(s, std::regex("'"), "'\\''") + "'";
-            };
-
             // Überhack to support Perl. Perl examines the shebang and
             // executes it unless it contains the string "perl" or "indir",
             // or (undocumented) argv[0] does not contain "perl". Exploit

src/nix-store/nix-store.cc

@@ -440,15 +440,6 @@ static void opQuery(Strings opFlags, Strings opArgs)
 }
 
 
-static string shellEscape(const string & s)
-{
-    string r;
-    for (auto & i : s)
-        if (i == '\'') r += "'\\''"; else r += i;
-    return r;
-}
-
-
 static void opPrintEnv(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
@@ -460,7 +451,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs)
     /* Print each environment variable in the derivation in a format
        that can be sourced by the shell. */
     for (auto & i : drv.env)
-        cout << format("export %1%; %1%='%2%'\n") % i.first % shellEscape(i.second);
+        cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second);
 
     /* Also output the arguments.  This doesn't preserve whitespace in
        arguments. */

tests/config.nix

@@ -13,7 +13,7 @@ rec {
     derivation ({
       inherit system;
       builder = shell;
-      args = ["-e" args.builder or (builtins.toFile "builder.sh" "eval \"$buildCommand\"")];
+      args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
       PATH = path;
     } // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };

tests/local.mk

@@ -14,7 +14,8 @@ nix_tests = \
   placeholders.sh nix-shell.sh \
   linux-sandbox.sh \
   build-remote.sh \
-  nar-index.sh
+  nar-index.sh \
+  structured-attrs.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

tests/structured-attrs.nix

@@ -0,0 +1,47 @@
+with import ./config.nix;
+
+mkDerivation {
+  name = "structured";
+
+  __structuredAttrs = true;
+
+  buildCommand = ''
+    set -x
+
+    [[ $int = 123456789 ]]
+    [[ -z $float ]]
+    [[ -n $boolTrue ]]
+    [[ -z $boolFalse ]]
+    [[ -n ''${hardening[format]} ]]
+    [[ -z ''${hardening[fortify]} ]]
+    [[ ''${#buildInputs[@]} = 7 ]]
+    [[ ''${buildInputs[2]} = c ]]
+    [[ -v nothing ]]
+    [[ -z $nothing ]]
+
+    mkdir ''${outputs[out]}
+    echo bar > $dest
+  '';
+
+  buildInputs = [ "a" "b" "c" 123 "'" "\"" null ];
+
+  hardening.format = true;
+  hardening.fortify = false;
+
+  outer.inner = [ 1 2 3 ];
+
+  int = 123456789;
+
+  float = 123.456;
+
+  boolTrue = true;
+  boolFalse = false;
+
+  nothing = null;
+
+  dest = "${placeholder "out"}/foo";
+
+  "foo bar" = "BAD";
+  "1foobar" = "BAD";
+  "foo$" = "BAD";
+}

tests/structured-attrs.sh

@@ -0,0 +1,7 @@
+source common.sh
+
+clearStore
+
+outPath=$(nix-build structured-attrs.nix --no-out-link)
+
+[[ $(cat $outPath/foo) = bar ]]