fix: remote python pack deployment

gauron99 · gauron99 · commit ce4e87550a83 · 2025-08-28T16:48:42.000+02:00
diff --git a/cmd/func-util/main.go b/cmd/func-util/main.go
@@ -87,7 +87,12 @@ func scaffold(ctx context.Context) error {
 	}
 
 	if f.Runtime != "go" && f.Runtime != "python" {
-		// Scaffolding is for now supported/needed only for Go.
+		// Scaffolding is for now supported/needed only for Go/Python
+		return nil
+	}
+
+	// special case for python-pack
+	if f.Build.Builder == "pack" && f.Runtime == "python" {
 		return nil
 	}
 
@@ -147,7 +152,7 @@ func deploy(ctx context.Context) error {
 			return fmt.Errorf("cannot determine working directory: %w", err)
 		}
 	}
-
+	fmt.Printf("root: %v\n", root)
 	f, err := fn.NewFunction(root)
 	if err != nil {
 		return fmt.Errorf("cannot load function: %w", err)
diff --git a/pkg/pipelines/tekton/pipelines_provider.go b/pkg/pipelines/tekton/pipelines_provider.go
@@ -272,6 +272,20 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader {
 		}
 	}
 
+	// Hack for python+pack where the function needs to be in a subdirectory
+	// and the actual main (scaffolding) needs to be in the source.
+	// So main in "root" and function in "root/subdir" otherwise pack builder
+	// determines that current function is JAVA or something because we currently
+	// build in the .s2i directory (needs fixing) and it contains a 'bin/' dir.
+	// buildpacks determines its somehow java.
+	//
+	// You can see this in the python scaffolding injector made for local builds
+	// for buildpacks builder & python runtime.
+	hName := "source"
+	usePyInjector := f.Build.Builder == "pack" && f.Runtime == "python"
+	if usePyInjector {
+		hName = "source/fn"
+	}
 	pr, pw := io.Pipe()
 
 	const nobodyID = 65534
@@ -282,7 +296,7 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader {
 
 		err := tw.WriteHeader(&tar.Header{
 			Typeflag: tar.TypeDir,
-			Name:     "source/",
+			Name:     hName,
 			Mode:     0777,
 			Uid:      nobodyID,
 			Gid:      nobodyID,
@@ -341,7 +355,8 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader {
 				return fmt.Errorf("cannot create a tar header: %w", err)
 			}
 			// "source" is expected path in workspace pvc
-			hdr.Name = path.Join("source", filepath.ToSlash(relp))
+			// current Hack: python + pack builder needs subdir ('source/fn')
+			hdr.Name = path.Join(hName, filepath.ToSlash(relp))
 
 			err = tw.WriteHeader(hdr)
 			if err != nil {
@@ -365,13 +380,61 @@ func sourcesAsTarStream(f fn.Function) *io.PipeReader {
 		if err != nil {
 			_ = pw.CloseWithError(fmt.Errorf("error while creating tar stream from sources: %w", err))
 		} else {
+			// python injector hack for python+pack builder
+			if usePyInjector {
+				err = pythonInjector(tw, f.Invoke)
+				if err != nil {
+					_ = pw.CloseWithError(fmt.Errorf("cannot inject python main into tar stream: %w", err))
+				}
+			}
 			_ = tw.Close()
 			_ = pw.Close()
 		}
 	}()
 	return pr
 }
 
+// inject python main etc. into the tar file at the root dir instead of the usual
+// subdir for python + pack builder
+func pythonInjector(tw *tar.Writer, invoke string) (err error) {
+	if invoke == "" {
+		invoke = "http"
+	}
+	// the function files were all written in "source/fn" therefore new header
+	// for the actual "source" is neeeded
+	for _, f := range []struct {
+		path    string
+		content string
+	}{
+		{
+			path:    "service/main.py",
+			content: mainContent(invoke),
+		},
+		{
+			path:    "pyproject.toml",
+			content: tomlContent,
+		},
+		{
+			path:    "service/__init__.py",
+			content: "",
+		},
+	} {
+		err := tw.WriteHeader(&tar.Header{
+			Name: "/source/" + f.path,
+			Size: int64(len(f.content)),
+			Mode: 0644,
+		})
+		if err != nil {
+			return err
+		}
+		_, err = tw.Write([]byte(f.content))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Remove tries to remove all resources that are present on the cluster and belongs to the input function and it's pipelines
 func (pp *PipelinesProvider) Remove(ctx context.Context, f fn.Function) error {
 	return pp.removeClusterResources(ctx, f)
@@ -574,3 +637,62 @@ func createPipelinePersistentVolumeClaim(ctx context.Context, f fn.Function, nam
 	}
 	return nil
 }
+
+func mainContent(invoke string) string {
+	template := `"""
+This code is glue between a user's Function and the middleware which will
+expose it as a network service.  This code is written on-demand when a
+Function is being built, deployed or run.  This will be included in the
+final container.
+"""
+import logging
+from func_python.%s import serve
+
+logging.basicConfig(level=logging.INFO)
+
+try:
+    from function import new as handler  # type: ignore[import]
+except ImportError:
+    try:
+        from function import handle as handler  # type: ignore[import]
+    except ImportError:
+        logging.error("Function must export either 'new' or 'handle'")
+        raise
+
+def main():
+    logging.info("Functions middleware invoking user function")
+    serve(handler)
+
+if __name__ == "__main__":
+    main()
+`
+	return fmt.Sprintf(template, invoke)
+}
+
+const tomlContent = `[project]
+name = "service"
+description = "an autogenerated service which runs a Function"
+version = "0.1.0"
+requires-python = ">=3.9"
+license = "MIT"
+dependencies = [
+    "func-python",
+    "function @ ./fn"
+]
+authors = [
+  { name="The Knative Authors", email="knative-dev@googlegroups.com"},
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.poetry.dependencies]
+python = ">=3.9,<4.0"
+
+[tool.poetry.scripts]
+script = "service.main:main"
+`
diff --git a/pkg/pipelines/tekton/tasks.go b/pkg/pipelines/tekton/tasks.go
@@ -62,8 +62,11 @@ spec:
       description: The registry associated with the function image.
     - name: BUILDER_IMAGE
       description: The image on which builds will run (must include lifecycle and compatible buildpacks).
-    - name: SOURCE_SUBPATH
-      description: A subpath within the "source" input where the source to build is located.
+    - name: contextDir
+      description: context directory for the function project
+      default: ""
+    - name: BUILD_TARGET
+      description: A directory to where to build from. (passed to pack builder)
       default: ""
     - name: ENV_VARS
       type: array
@@ -157,10 +160,10 @@ spec:
         ############################################
 
         func_file="$(workspaces.source.path)/func.yaml"
-        if [ "$(params.SOURCE_SUBPATH)" != "" ]; then
-          func_file="$(workspaces.source.path)/$(params.SOURCE_SUBPATH)/func.yaml"
+        if [ "$(params.contextDir)" != "" ]; then
+          func_file="$(workspaces.source.path)/$(params.contextDir)/func.yaml"
         fi
-        echo "--> Saving 'func.yaml'"
+        echo "--> Saving 'func.yaml' from '$func_file'"
         cp $func_file /emptyDir/func.yaml
 
         ############################################
@@ -183,7 +186,7 @@ spec:
         - name: DOCKER_CONFIG
           value: $(workspaces.dockerconfig.path)
       args:
-        - "-app=$(workspaces.source.path)/$(params.SOURCE_SUBPATH)"
+        - "-app=$(workspaces.source.path)/$(params.BUILD_TARGET)"
         - "-cache-dir=$(workspaces.cache.path)"
         - "-cache-image=$(params.CACHE_IMAGE)"
         - "-uid=$(params.USER_ID)"
@@ -220,13 +223,13 @@ spec:
         digest=$(cat $(results.IMAGE_DIGEST.path))
 
         func_file="$(workspaces.source.path)/func.yaml"
-        if [ "$(params.SOURCE_SUBPATH)" != "" ]; then
-          func_file="$(workspaces.source.path)/$(params.SOURCE_SUBPATH)/func.yaml"
+        if [ "$(params.contextDir)" != "" ]; then
+          func_file="$(workspaces.source.path)/$(params.contextDir)/func.yaml"
         fi
 
         if [[ ! -f "$func_file" ]]; then
           echo "--> Restoring 'func.yaml'"
-          mkdir -p "$(workspaces.source.path)/$(params.SOURCE_SUBPATH)"
+          mkdir -p "$(workspaces.source.path)/$(params.contextDir)"
           cp /emptyDir/func.yaml $func_file
         fi
 
@@ -413,13 +416,16 @@ spec:
     - name: image
       description: Container image to be deployed
       default: ""
+    - name: subpath
+      default: ""
+      description: Optional Subpath to where func.yaml is
   workspaces:
     - name: source
       description: The workspace containing the function project
   steps:
     - name: func-deploy
       image: "%s"
-      command: ["deploy", "$(params.path)", "$(params.image)"]
+      command: ["deploy", "$(params.path)", "$(params.image)", "$(params.subpath)"]
 `, DeployerImage)
 }
 
diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
 	"text/template"
 
@@ -77,14 +78,15 @@ const (
 )
 
 type templateData struct {
-	FunctionName  string
-	Annotations   map[string]string
-	Labels        map[string]string
-	ContextDir    string
-	FunctionImage string
-	Registry      string
-	BuilderImage  string
-	BuildEnvs     []string
+	FunctionName    string
+	Annotations     map[string]string
+	Labels          map[string]string
+	ContextDir      string
+	FunctionImage   string
+	SubPathOverride string
+	Registry        string
+	BuilderImage    string
+	BuildEnvs       []string
 
 	PipelineName    string
 	PipelineRunName string
@@ -387,14 +389,15 @@ func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels m
 	}
 
 	data := templateData{
-		FunctionName:  f.Name,
-		Annotations:   f.Deploy.Annotations,
-		Labels:        labels,
-		ContextDir:    contextDir,
-		FunctionImage: f.Deploy.Image,
-		Registry:      f.Registry,
-		BuilderImage:  getBuilderImage(f),
-		BuildEnvs:     buildEnvs,
+		FunctionName:    f.Name,
+		Annotations:     f.Deploy.Annotations,
+		Labels:          labels,
+		ContextDir:      contextDir,
+		SubPathOverride: contextDir,
+		FunctionImage:   f.Deploy.Image,
+		Registry:        f.Registry,
+		BuilderImage:    getBuilderImage(f),
+		BuildEnvs:       buildEnvs,
 
 		PipelineName:    getPipelineName(f),
 		PipelineRunName: getPipelineRunGenerateName(f),
@@ -407,6 +410,13 @@ func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels m
 		Revision: pipelinesTargetBranch,
 	}
 
+	// this is for current impl. of python+pack hack
+	// its for pack builder which needs the path of where to deploy and its
+	// different from where func.yaml is
+	if f.Runtime == "python" && f.Build.Builder == "pack" {
+		data.ContextDir = filepath.Join(data.ContextDir, "fn")
+	}
+
 	var template string
 	switch f.Build.Builder {
 	case builders.Pack:
diff --git a/pkg/pipelines/tekton/templates_pack.go b/pkg/pipelines/tekton/templates_pack.go
@@ -28,6 +28,9 @@ spec:
       description: Path where the function project is
       name: contextDir
       type: string
+    - default: ''
+      name: subPathOverride
+      description: Dir to build from (might not be where func.yaml is)
     - description: Function image name
       name: imageName
       type: string
@@ -57,8 +60,10 @@ spec:
           value: $(params.imageName)
         - name: REGISTRY
           value: $(params.registry)
-        - name: SOURCE_SUBPATH
+        - name: contextDir
           value: $(params.contextDir)
+        - name: SOURCE_SUBPATH
+          value: $(params.subPathOverride)
         - name: BUILDER_IMAGE
           value: $(params.builderImage)
         - name: ENV_VARS
@@ -120,6 +125,8 @@ spec:
       value: {{.Revision}}
     - name: contextDir
       value: "{{.ContextDir}}"
+    - name: subPathOverride
+      value: "{{.SubPathOverride}}"
     - name: imageName
       value: {{.FunctionImage}}
     - name: registry

Original file line number	Diff line number	Diff line change
`@@ -87,7 +87,12 @@ func scaffold(ctx context.Context) error {`
`87`	`87`	`}`
`88`	`88`
`89`	`89`	`if f.Runtime != "go" && f.Runtime != "python" {`
`90`		`- // Scaffolding is for now supported/needed only for Go.`
	`90`	`+ // Scaffolding is for now supported/needed only for Go/Python`
	`91`	`+ return nil`
	`92`	`+ }`
	`93`	`+`
	`94`	`+ // special case for python-pack`
	`95`	`+ if f.Build.Builder == "pack" && f.Runtime == "python" {`
`91`	`96`	`return nil`
`92`	`97`	`}`
`93`	`98`
`@@ -147,7 +152,7 @@ func deploy(ctx context.Context) error {`
`147`	`152`	`return fmt.Errorf("cannot determine working directory: %w", err)`
`148`	`153`	`}`
`149`	`154`	`}`
`150`		`-`
	`155`	`+ fmt.Printf("root: %v\n", root)`
`151`	`156`	`f, err := fn.NewFunction(root)`
`152`	`157`	`if err != nil {`
`153`	`158`	`return fmt.Errorf("cannot load function: %w", err)`