fix: remote python pack deployment

Author: gauron99

cmd/func-util/main.go

@@ -87,7 +87,12 @@ func scaffold(ctx context.Context) error {
 	}
 
 	if f.Runtime != "go" && f.Runtime != "python" {
-		// Scaffolding is for now supported/needed only for Go.
+		// Scaffolding is for now supported/needed only for Go/Python
+		return nil
+	}
+
+	// special case for python-pack
+	if f.Build.Builder == "pack" && f.Runtime == "python" {
 		return nil
 	}
 
@@ -147,7 +152,10 @@ func deploy(ctx context.Context) error {
 			return fmt.Errorf("cannot determine working directory: %w", err)
 		}
 	}
-
+	// optional path definition to where the func.yaml is. Only relevant for
+	// python+pack, otherwise empty
+	root = filepath.Join(root, os.Args[3])
+	fmt.Printf("root: %v\n", root)
 	f, err := fn.NewFunction(root)
 	if err != nil {
 		return fmt.Errorf("cannot load function: %w", err)

pkg/pipelines/tekton/pipelines_provider.go

@@ -272,6 +272,20 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader {
 		}
 	}
 
+	// Hack for python+pack where the function needs to be in a subdirectory
+	// and the actual main (scaffolding) needs to be in the source.
+	// So main in "root" and function in "root/subdir" otherwise pack builder
+	// determines that current function is JAVA or something because we currently
+	// build in the .s2i directory (needs fixing) and it contains a 'bin/' dir.
+	// buildpacks determines its somehow java.
+	//
+	// You can see this in the python scaffolding injector made for local builds
+	// for buildpacks builder & python runtime.
+	hName := "source"
+	usePyInjector := f.Build.Builder == "pack" && f.Runtime == "python"
+	if usePyInjector {
+		hName = "source/fn"
+	}
 	pr, pw := io.Pipe()
 
 	const nobodyID = 65534
@@ -282,7 +296,7 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader {
 
 		err := tw.WriteHeader(&tar.Header{
 			Typeflag: tar.TypeDir,
-			Name:     "source/",
+			Name:     hName,
 			Mode:     0777,
 			Uid:      nobodyID,
 			Gid:      nobodyID,
@@ -341,7 +355,8 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader {
 				return fmt.Errorf("cannot create a tar header: %w", err)
 			}
 			// "source" is expected path in workspace pvc
-			hdr.Name = path.Join("source", filepath.ToSlash(relp))
+			// current Hack: python + pack builder needs subdir ('source/fn')
+			hdr.Name = path.Join(hName, filepath.ToSlash(relp))
 
 			err = tw.WriteHeader(hdr)
 			if err != nil {
@@ -365,13 +380,61 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader {
 		if err != nil {
 			_ = pw.CloseWithError(fmt.Errorf("error while creating tar stream from sources: %w", err))
 		} else {
+			// python injector hack for python+pack builder
+			if usePyInjector {
+				err = pythonInjector(tw, f.Invoke)
+				if err != nil {
+					_ = pw.CloseWithError(fmt.Errorf("cannot inject python main into tar stream: %w", err))
+				}
+			}
 			_ = tw.Close()
 			_ = pw.Close()
 		}
 	}()
 	return pr
 }
 
+// inject python main etc. into the tar file at the root dir instead of the usual
+// subdir for python + pack builder
+func pythonInjector(tw *tar.Writer, invoke string) (err error) {
+	if invoke == "" {
+		invoke = "http"
+	}
+	// the function files were all written in "source/fn" therefore new header
+	// for the actual "source" is neeeded
+	for _, f := range []struct {
+		path    string
+		content string
+	}{
+		{
+			path:    "service/main.py",
+			content: mainContent(invoke),
+		},
+		{
+			path:    "pyproject.toml",
+			content: tomlContent,
+		},
+		{
+			path:    "service/__init__.py",
+			content: "",
+		},
+	} {
+		err := tw.WriteHeader(&tar.Header{
+			Name: "/source/" + f.path,
+			Size: int64(len(f.content)),
+			Mode: 0644,
+		})
+		if err != nil {
+			return err
+		}
+		_, err = tw.Write([]byte(f.content))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Remove tries to remove all resources that are present on the cluster and belongs to the input function and it's pipelines
 func (pp *PipelinesProvider) Remove(ctx context.Context, f fn.Function) error {
 	return pp.removeClusterResources(ctx, f)
@@ -574,3 +637,62 @@ func createPipelinePersistentVolumeClaim(ctx context.Context, f fn.Function, nam
 	}
 	return nil
 }
+
+func mainContent(invoke string) string {
+	template := `"""
+This code is glue between a user's Function and the middleware which will
+expose it as a network service.  This code is written on-demand when a
+Function is being built, deployed or run.  This will be included in the
+final container.
+"""
+import logging
+from func_python.%s import serve
+
+logging.basicConfig(level=logging.INFO)
+
+try:
+    from function import new as handler  # type: ignore[import]
+except ImportError:
+    try:
+        from function import handle as handler  # type: ignore[import]
+    except ImportError:
+        logging.error("Function must export either 'new' or 'handle'")
+        raise
+
+def main():
+    logging.info("Functions middleware invoking user function")
+    serve(handler)
+
+if __name__ == "__main__":
+    main()
+`
+	return fmt.Sprintf(template, invoke)
+}
+
+const tomlContent = `[project]
+name = "service"
+description = "an autogenerated service which runs a Function"
+version = "0.1.0"
+requires-python = ">=3.9"
+license = "MIT"
+dependencies = [
+    "func-python",
+    "function @ ./fn"
+]
+authors = [
+  { name="The Knative Authors", email="[email protected]"},
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.poetry.dependencies]
+python = ">=3.9,<4.0"
+
+[tool.poetry.scripts]
+script = "service.main:main"
+`

pkg/pipelines/tekton/tasks.go

@@ -62,9 +62,12 @@ spec:
       description: The registry associated with the function image.
     - name: BUILDER_IMAGE
       description: The image on which builds will run (must include lifecycle and compatible buildpacks).
+    - name: contextDir
+      description: context directory for the function project
+      default: ""
     - name: SOURCE_SUBPATH
       description: A subpath within the "source" input where the source to build is located.
-      default: ""
+      default: "$(params.contextDir)"
     - name: ENV_VARS
       type: array
       description: Environment variables to set during _build-time_.
@@ -160,7 +163,7 @@ spec:
         if [ "$(params.SOURCE_SUBPATH)" != "" ]; then
           func_file="$(workspaces.source.path)/$(params.SOURCE_SUBPATH)/func.yaml"
         fi
-        echo "--> Saving 'func.yaml'"
+        echo "--> Saving 'func.yaml' from '$func_file'"
         cp $func_file /emptyDir/func.yaml
 
         ############################################
@@ -183,7 +186,7 @@ spec:
         - name: DOCKER_CONFIG
           value: $(workspaces.dockerconfig.path)
       args:
-        - "-app=$(workspaces.source.path)/$(params.SOURCE_SUBPATH)"
+        - "-app=$(workspaces.source.path)/$(params.contextDir)"
         - "-cache-dir=$(workspaces.cache.path)"
         - "-cache-image=$(params.CACHE_IMAGE)"
         - "-uid=$(params.USER_ID)"
@@ -413,13 +416,16 @@ spec:
     - name: image
       description: Container image to be deployed
       default: ""
+    - name: subpath
+      default: ""
+      description: Optional Subpath to where func.yaml is
   workspaces:
     - name: source
       description: The workspace containing the function project
   steps:
     - name: func-deploy
       image: "%s"
-      command: ["deploy", "$(params.path)", "$(params.image)"]
+      command: ["deploy", "$(params.path)", "$(params.image)", "$(params.subpath)"]
 `, DeployerImage)
 }
 

pkg/pipelines/tekton/templates.go

@@ -77,14 +77,15 @@ const (
 )
 
 type templateData struct {
-	FunctionName  string
-	Annotations   map[string]string
-	Labels        map[string]string
-	ContextDir    string
-	FunctionImage string
-	Registry      string
-	BuilderImage  string
-	BuildEnvs     []string
+	FunctionName    string
+	Annotations     map[string]string
+	Labels          map[string]string
+	ContextDir      string
+	FunctionImage   string
+	SubPathOverride string
+	Registry        string
+	BuilderImage    string
+	BuildEnvs       []string
 
 	PipelineName    string
 	PipelineRunName string
@@ -407,6 +408,11 @@ func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels m
 		Revision: pipelinesTargetBranch,
 	}
 
+	// func project is gonna be in a subdirectory
+	if f.Runtime == "python" && f.Build.Builder == "pack" {
+		data.SubPathOverride = "fn"
+	}
+
 	var template string
 	switch f.Build.Builder {
 	case builders.Pack:

pkg/pipelines/tekton/templates_pack.go

@@ -28,6 +28,9 @@ spec:
       description: Path where the function project is
       name: contextDir
       type: string
+    - default: ''
+      name: subPathOverride
+      description: Override for subpath
     - description: Function image name
       name: imageName
       type: string
@@ -58,7 +61,7 @@ spec:
         - name: REGISTRY
           value: $(params.registry)
         - name: SOURCE_SUBPATH
-          value: $(params.contextDir)
+          value: $(params.subPathOverride)
         - name: BUILDER_IMAGE
           value: $(params.builderImage)
         - name: ENV_VARS
@@ -80,6 +83,8 @@ spec:
           value: $(workspaces.source.path)/$(params.contextDir)
         - name: image
           value: $(params.imageName)@$(tasks.build.results.IMAGE_DIGEST)
+        - name: subpath
+          value: $(params.subPathOverride)
       runAfter:
         - build
       {{.FuncDeployTaskRef}}
@@ -120,6 +125,8 @@ spec:
       value: {{.Revision}}
     - name: contextDir
       value: "{{.ContextDir}}"
+    - name: subPathOverride
+      value: "{{.SubPathOverride}}"
     - name: imageName
       value: {{.FunctionImage}}
     - name: registry