[v4.7] Fix env-file regression + more backports

docs/source/Reference.rst

@@ -7,6 +7,8 @@ Show the API documentation for version:
 
 * `latest (main branch) <_static/api.html>`_
 
+* `version 4.7 <_static/api.html?version=v4.7>`_
+
 * `version 4.6 <_static/api.html?version=v4.6>`_
 
 * `version 4.5 <_static/api.html?version=v4.5>`_

docs/source/markdown/options/env-file.md

@@ -4,8 +4,4 @@
 ####> are applicable to all of those.
 #### **--env-file**=*file*
 
-Read the environment variables from the file, supporting prefix matching: `KEY*`, as well as multiline values in double quotes and single quotes, but not multiline values in backticks.
-The env-file will ignore comments and empty lines. And spaces or tabs before and after the KEY.
-If an invalid value is encountered, such as only an `=` sign, it will be skipped. If it is a prefix match (`KEY*`), all environment variables starting with KEY on the host machine will be loaded.
-If it is only KEY (`KEY`), the KEY environment variable on the host machine will be loaded.
-Compatible with the `export` syntax in **dotenv**, such as: `export KEY=bar`.
+Read in a line-delimited file of environment variables.

libpod/container.go

@@ -1356,6 +1356,21 @@ func (d ContainerNetworkDescriptions) getInterfaceByName(networkName string) (st
 	return fmt.Sprintf("eth%d", val), exists
 }
 
+// GetNetworkStatus returns the current network status for this container.
+// This returns a map without deep copying which means this should only ever
+// be used as read only access, do not modify this status.
+func (c *Container) GetNetworkStatus() (map[string]types.StatusBlock, error) {
+	if !c.batched {
+		c.lock.Lock()
+		defer c.lock.Unlock()
+
+		if err := c.syncContainer(); err != nil {
+			return nil, err
+		}
+	}
+	return c.getNetworkStatus(), nil
+}
+
 // getNetworkStatus get the current network status from the state. If the container
 // still uses the old network status it is converted to the new format. This function
 // should be used instead of reading c.state.NetworkStatus directly.

pkg/api/handlers/compat/networks.go

@@ -22,6 +22,36 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+type containerNetStatus struct {
+	name   string
+	id     string
+	status map[string]nettypes.StatusBlock
+}
+
+func getContainerNetStatuses(rt *libpod.Runtime) ([]containerNetStatus, error) {
+	cons, err := rt.GetAllContainers()
+	if err != nil {
+		return nil, err
+	}
+	statuses := make([]containerNetStatus, 0, len(cons))
+	for _, con := range cons {
+		status, err := con.GetNetworkStatus()
+		if err != nil {
+			if errors.Is(err, define.ErrNoSuchCtr) || errors.Is(err, define.ErrCtrRemoved) {
+				continue
+			}
+			return nil, err
+		}
+
+		statuses = append(statuses, containerNetStatus{
+			id:     con.ID(),
+			name:   con.Name(),
+			status: status,
+		})
+	}
+	return statuses, nil
+}
+
 func normalizeNetworkName(rt *libpod.Runtime, name string) (string, bool) {
 	if name == nettypes.BridgeNetworkDriver {
 		return rt.Network().DefaultNetworkName(), true
@@ -56,45 +86,42 @@ func InspectNetwork(w http.ResponseWriter, r *http.Request) {
 		utils.NetworkNotFound(w, name, err)
 		return
 	}
-	report, err := convertLibpodNetworktoDockerNetwork(runtime, &net, changed)
+	statuses, err := getContainerNetStatuses(runtime)
 	if err != nil {
 		utils.InternalServerError(w, err)
 		return
 	}
+	report := convertLibpodNetworktoDockerNetwork(runtime, statuses, &net, changed)
 	utils.WriteResponse(w, http.StatusOK, report)
 }
 
-func convertLibpodNetworktoDockerNetwork(runtime *libpod.Runtime, network *nettypes.Network, changeDefaultName bool) (*types.NetworkResource, error) {
-	cons, err := runtime.GetAllContainers()
-	if err != nil {
-		return nil, err
-	}
-	containerEndpoints := make(map[string]types.EndpointResource, len(cons))
-	for _, con := range cons {
-		data, err := con.Inspect(false)
-		if err != nil {
-			if errors.Is(err, define.ErrNoSuchCtr) || errors.Is(err, define.ErrCtrRemoved) {
-				continue
-			}
-			return nil, err
-		}
-		if netData, ok := data.NetworkSettings.Networks[network.Name]; ok {
+func convertLibpodNetworktoDockerNetwork(runtime *libpod.Runtime, statuses []containerNetStatus, network *nettypes.Network, changeDefaultName bool) *types.NetworkResource {
+	containerEndpoints := make(map[string]types.EndpointResource, len(statuses))
+	for _, st := range statuses {
+		if netData, ok := st.status[network.Name]; ok {
 			ipv4Address := ""
-			if netData.IPAddress != "" {
-				ipv4Address = fmt.Sprintf("%s/%d", netData.IPAddress, netData.IPPrefixLen)
-			}
 			ipv6Address := ""
-			if netData.GlobalIPv6Address != "" {
-				ipv6Address = fmt.Sprintf("%s/%d", netData.GlobalIPv6Address, netData.GlobalIPv6PrefixLen)
+			macAddr := ""
+			for _, dev := range netData.Interfaces {
+				for _, subnet := range dev.Subnets {
+					// Note the docker API really wants the full CIDR subnet not just a single ip.
+					// https://github.com/containers/podman/pull/12328
+					if netutil.IsIPv4(subnet.IPNet.IP) {
+						ipv4Address = subnet.IPNet.String()
+					} else {
+						ipv6Address = subnet.IPNet.String()
+					}
+				}
+				macAddr = dev.MacAddress.String()
+				break
 			}
 			containerEndpoint := types.EndpointResource{
-				Name:        con.Name(),
-				EndpointID:  netData.EndpointID,
-				MacAddress:  netData.MacAddress,
+				Name:        st.name,
+				MacAddress:  macAddr,
 				IPv4Address: ipv4Address,
 				IPv6Address: ipv6Address,
 			}
-			containerEndpoints[con.ID()] = containerEndpoint
+			containerEndpoints[st.id] = containerEndpoint
 		}
 	}
 	ipamConfigs := make([]dockerNetwork.IPAMConfig, 0, len(network.Subnets))
@@ -144,7 +171,7 @@ func convertLibpodNetworktoDockerNetwork(runtime *libpod.Runtime, network *netty
 		Peers:      nil,
 		Services:   nil,
 	}
-	return &report, nil
+	return &report
 }
 
 func ListNetworks(w http.ResponseWriter, r *http.Request) {
@@ -165,13 +192,14 @@ func ListNetworks(w http.ResponseWriter, r *http.Request) {
 		utils.InternalServerError(w, err)
 		return
 	}
+	statuses, err := getContainerNetStatuses(runtime)
+	if err != nil {
+		utils.InternalServerError(w, err)
+		return
+	}
 	reports := make([]*types.NetworkResource, 0, len(nets))
 	for _, net := range nets {
-		report, err := convertLibpodNetworktoDockerNetwork(runtime, &net, true)
-		if err != nil {
-			utils.InternalServerError(w, err)
-			return
-		}
+		report := convertLibpodNetworktoDockerNetwork(runtime, statuses, &net, true)
 		reports = append(reports, report)
 	}
 	utils.WriteResponse(w, http.StatusOK, reports)

pkg/env/env.go

@@ -1,29 +1,13 @@
+// Package for processing environment variables.
 package env
 
+// TODO: we need to add tests for this package.
+
 import (
+	"bufio"
 	"fmt"
-	"io"
 	"os"
 	"strings"
-
-	"github.com/containers/storage/pkg/regexp"
-)
-
-var (
-	// Form: https://github.com/motdotla/dotenv/blob/aa03dcad1002027390dac1e8d96ac236274de354/lib/main.js#L9C76-L9C76
-	// (?:export\s+)?([\w.-]+) match key
-	// ([\w.%-]+)(\s*[=|*]\s*?|:\s+?) match separator
-	// Remaining match value
-	// e.g. KEY=VALUE => KEY, =, VALUE
-	//
-	//	KEY= => KEY, =, ""
-	//	KEY* => KEY, *, ""
-	//	KEY*=1 => KEY, *, =1
-	lineRegexp = regexp.Delayed(
-		`(?m)(?:^|^)\s*(?:export\s+)?([\w.%-]+)(\s*[=|*]\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*` +
-			"`(?:\\`|[^`])*`" + `|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)`,
-	)
-	onlyKeyRegexp = regexp.Delayed(`^[\w.-]+$`)
 )
 
 const whiteSpaces = " \t"
@@ -95,120 +79,26 @@ func ParseFile(path string) (_ map[string]string, err error) {
 	}
 	defer fh.Close()
 
-	content, err := io.ReadAll(fh)
-	if err != nil {
-		return nil, err
-	}
-
-	// replace all \r\n and \r with \n
-	text := strings.NewReplacer("\r\n", "\n", "\r", "\n").Replace(string(content))
-	if err := parseEnv(env, text); err != nil {
-		return nil, err
-	}
-
-	return env, nil
-}
-
-// parseEnv parse the given content into env format
-//
-// @example: parseEnv(env, "#comment") => nil
-// @example: parseEnv(env, "") => nil
-// @example: parseEnv(env, "KEY=FOO") => nil
-// @example: parseEnv(env, "KEY") => nil
-func parseEnv(env map[string]string, content string) error {
-	m := envMatch(content)
-
-	for _, match := range m {
-		key := match[1]
-		separator := strings.Trim(match[2], whiteSpaces)
-		value := match[3]
-
-		if strings.Contains(value, "\n") {
-			if strings.HasPrefix(value, "`") {
-				return fmt.Errorf("only support multi-line environment variables surrounded by "+
-					"double quotation marks or single quotation marks. invalid variable: %q", match[0])
-			}
-
-			// In the case of multi-line values, we need to remove the surrounding " '
-			value = strings.Trim(value, "\"'")
-		}
-
-		// KEY*=1 => KEY, *, =1 => KEY*, =, 1
-		if separator == "*" && strings.HasPrefix(value, "=") {
-			key += "*"
-			separator = "="
-			value = strings.TrimPrefix(value, "=")
-		}
-
-		switch separator {
-		case "=":
-			// KEY=
-			if value == "" {
-				if val, ok := os.LookupEnv(key); ok {
-					env[key] = val
-				}
-			} else {
-				env[key] = value
-			}
-		case "*":
-			for _, e := range os.Environ() {
-				part := strings.SplitN(e, "=", 2)
-				if len(part) < 2 {
-					continue
-				}
-				if strings.HasPrefix(part[0], key) {
-					env[part[0]] = part[1]
-				}
+	scanner := bufio.NewScanner(fh)
+	for scanner.Scan() {
+		// trim the line from all leading whitespace first
+		line := strings.TrimLeft(scanner.Text(), whiteSpaces)
+		// line is not empty, and not starting with '#'
+		if len(line) > 0 && !strings.HasPrefix(line, "#") {
+			if err := parseEnv(env, line); err != nil {
+				return nil, err
 			}
 		}
 	}
-	return nil
-}
-
-func envMatch(content string) [][]string {
-	m := lineRegexp.FindAllStringSubmatch(content, -1)
-
-	// KEY => KEY, =, ""
-	// Due to the above regex pattern, it will skip cases where only KEY is present (e.g., foo).
-	// However, in our requirement, this situation is equivalent to foo=(i.e., "foo" == "foo=").
-	// Therefore, we need to perform additional processing.
-	// The reason for needing to support this scenario is that we need to consider: `podman run -e CI -e USERNAME`.
-	{
-		noMatched := lineRegexp.ReplaceAllString(content, "")
-		nl := strings.Split(noMatched, "\n")
-		for _, key := range nl {
-			key := strings.Trim(key, whiteSpaces)
-			if key == "" {
-				continue
-			}
-			if onlyKeyRegexp.MatchString(key) {
-				m = append(m, []string{key, key, "=", ""})
-			}
-		}
-	}
-
-	return m
+	return env, scanner.Err()
 }
 
-// parseEnvWithSlice parsing a set of Env variables from a slice of strings
-// because the majority of shell interpreters discard double quotes and single quotes,
-// for example: podman run -e K='V', when passed into a program, it will become: K=V.
-// This can lead to unexpected issues, as discussed in this link: https://github.com/containers/podman/pull/19096#issuecomment-1670164724.
-//
-// parseEnv method will discard all comments (#) that are not wrapped in quotation marks,
-// so it cannot be used to parse env variables obtained from the command line.
-//
-// @example: parseEnvWithSlice(env, "KEY=FOO") => KEY: FOO
-// @example: parseEnvWithSlice(env, "KEY") => KEY: ""
-// @example: parseEnvWithSlice(env, "KEY=") => KEY: ""
-// @example: parseEnvWithSlice(env, "KEY=FOO=BAR") => KEY: FOO=BAR
-// @example: parseEnvWithSlice(env, "KEY=FOO#BAR") => KEY: FOO#BAR
-func parseEnvWithSlice(env map[string]string, content string) error {
-	data := strings.SplitN(content, "=", 2)
+func parseEnv(env map[string]string, line string) error {
+	data := strings.SplitN(line, "=", 2)
 
 	// catch invalid variables such as "=" or "=A"
 	if data[0] == "" {
-		return fmt.Errorf("invalid variable: %q", content)
+		return fmt.Errorf("invalid variable: %q", line)
 	}
 	// trim the front of a variable, but nothing else
 	name := strings.TrimLeft(data[0], whiteSpaces)