internal/scorecard/alpha: add parallelism support

changelog/fragments/scorecard-parallel.yaml

@@ -0,0 +1,55 @@
+entries:
+  - description: >
+      Changes the scorecard configuration format to include a new top-level
+      `stages` field, which allows users to define stages of tests and to
+      enable parallelism within each stage.
+
+    kind: "change"
+
+    # Is this a breaking change?
+    breaking: true
+
+    # NOTE: ONLY USE `pull_request_override` WHEN ADDING THIS
+    # FILE FOR A PREVIOUSLY MERGED PULL_REQUEST!
+    #
+    # The generator auto-detects the PR number from the commit
+    # message in which this file was originally added.
+    #
+    # What is the pull request number (without the "#")?
+    # pull_request_override: 0
+
+
+    # Migration can be defined to automatically add a section to
+    # the migration guide. This is required for breaking changes.
+    migration:
+      header: Add stages to scorecard configuration file
+      body: |
+        Update your scorecard configuration file to contain a new top-level
+        stages field, and move your test definitions under one or more stages.
+
+        **Example**
+
+        ```yaml
+        tests:
+        - image: quay.io/operator-framework/scorecard-test:dev
+          entrypoint:
+          - scorecard-test
+          - basic-check-spec
+          labels:
+            suite: basic
+            test: basic-check-spec-test
+        ```
+
+        should be converted to
+
+        ```yaml
+        stages:
+        - tests:
+          - image: quay.io/operator-framework/scorecard-test:dev
+            entrypoint:
+            - scorecard-test
+            - basic-check-spec
+            labels:
+              suite: basic
+              test: basic-check-spec-test
+        ```

internal/registry/labels.go

@@ -27,7 +27,7 @@ import (
 	registrybundle "github.com/operator-framework/operator-registry/pkg/lib/bundle"
 	log "github.com/sirupsen/logrus"
 
-	// TODO: replace `gopkg.in/yaml.v2` with `sigs.k8s.io/yaml` once operator-registry has `json` tags in the
+	// TODO: replace `gopkg.in/yaml.v3` with `sigs.k8s.io/yaml` once operator-registry has `json` tags in the
 	// annotations struct.
 	yaml "gopkg.in/yaml.v3"
 )

internal/scorecard/alpha/config.go

@@ -25,6 +25,11 @@ const (
 	ConfigDirPath = "/tests/" + ConfigDirName + "/"
 )
 
+type Stage struct {
+	Parallel bool   `yaml:"parallel"`
+	Tests    []Test `yaml:"tests"`
+}
+
 type Test struct {
 	// Image is the name of the testimage
 	Image string `json:"image"`
@@ -37,7 +42,7 @@ type Test struct {
 // Config represents the set of test configurations which scorecard
 // would run based on user input
 type Config struct {
-	Tests []Test `yaml:"tests"`
+	Stages []Stage `yaml:"stages"`
 }
 
 // LoadConfig will find and return the scorecard config, the config file

internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml

@@ -1,15 +1,16 @@
-tests:
-- image: quay.io/username/custom-scorecard-tests:dev
-  entrypoint:
-  - custom-scorecard-tests
-  - customtest1
-  labels:
-    suite: custom
-    test: customtest1
-- image: quay.io/username/custom-scorecard-tests:dev
-  entrypoint:
-  - custom-scorecard-tests
-  - customtest2
-  labels:
-    suite: custom
-    test: customtest2
+stages:
+- tests:
+  - image: quay.io/username/custom-scorecard-tests:dev
+    entrypoint:
+    - custom-scorecard-tests
+    - customtest1
+    labels:
+      suite: custom
+      test: customtest1
+  - image: quay.io/username/custom-scorecard-tests:dev
+    entrypoint:
+    - custom-scorecard-tests
+    - customtest2
+    labels:
+      suite: custom
+      test: customtest2

internal/scorecard/alpha/formatting.go

@@ -42,17 +42,17 @@ func (r PodTestRunner) getTestStatus(ctx context.Context, p *v1.Pod) (output *v1
 // run based on user selection
 func (o Scorecard) List() v1alpha3.TestList {
 	output := v1alpha3.NewTestList()
-	tests := o.selectTests()
-
-	for _, test := range tests {
-		item := v1alpha3.NewTest()
-		item.Spec = v1alpha3.TestSpec{
-			Image:      test.Image,
-			Entrypoint: test.Entrypoint,
-			Labels:     test.Labels,
+	for _, stage := range o.Config.Stages {
+		tests := o.selectTests(stage)
+		for _, test := range tests {
+			item := v1alpha3.NewTest()
+			item.Spec = v1alpha3.TestSpec{
+				Image:      test.Image,
+				Entrypoint: test.Entrypoint,
+				Labels:     test.Labels,
+			}
+			output.Items = append(output.Items, item)
 		}
-		output.Items = append(output.Items, item)
 	}
-
 	return output
 }

internal/scorecard/alpha/labels_test.go

@@ -55,7 +55,7 @@ func TestEmptySelector(t *testing.T) {
 				return
 			}
 
-			tests := o.selectTests()
+			tests := o.selectTests(o.Config.Stages[0])
 			testsSelected := len(tests)
 			if testsSelected != c.testsSelected {
 				t.Errorf("Wanted testsSelected %d, got: %d", c.testsSelected, testsSelected)
@@ -65,51 +65,52 @@ func TestEmptySelector(t *testing.T) {
 	}
 }
 
-const testConfig = `tests:
-- image: quay.io/someuser/customtest1:v0.0.1
-  entrypoint: 
-  - custom-test
-  labels:
-    suite: custom
-    test: customtest1
-- image: quay.io/someuser/customtest2:v0.0.1
-  entrypoint: 
-  - custom-test
-  labels:
-    suite: custom
-    test: customtest2
-- image: quay.io/redhat/basictests:v0.0.1
-  entrypoint: 
-  - scorecard-test
-  - basic-check-spec
-  labels:
-    suite: basic
-    test: basic-check-spec-test
-- image: quay.io/redhat/basictests:v0.0.1
-  entrypoint: 
-  - scorecard-test
-  - basic-check-status
-  labels:
-    suite: basic
-    test: basic-check-status-test
-- image: quay.io/redhat/olmtests:v0.0.1
-  entrypoint: 
-  - scorecard-test
-  - olm-bundle-validation
-  labels:
-    suite: olm
-    test: olm-bundle-validation-test
-- image: quay.io/redhat/olmtests:v0.0.1
-  entrypoint: 
-  - scorecard-test
-  - olm-crds-have-validation
-  labels:
-    suite: olm
-    test: olm-crds-have-validation-test
-- image: quay.io/redhat/kuttltests:v0.0.1
-  labels:
-    suite: kuttl
-  entrypoint:
-  - kuttl-test
-  - olm-status-descriptors
-  `
+const testConfig = `stages:
+- tests:
+  - image: quay.io/someuser/customtest1:v0.0.1
+    entrypoint: 
+    - custom-test
+    labels:
+      suite: custom
+      test: customtest1
+  - image: quay.io/someuser/customtest2:v0.0.1
+    entrypoint: 
+    - custom-test
+    labels:
+      suite: custom
+      test: customtest2
+  - image: quay.io/redhat/basictests:v0.0.1
+    entrypoint: 
+    - scorecard-test
+    - basic-check-spec
+    labels:
+      suite: basic
+      test: basic-check-spec-test
+  - image: quay.io/redhat/basictests:v0.0.1
+    entrypoint: 
+    - scorecard-test
+    - basic-check-status
+    labels:
+      suite: basic
+      test: basic-check-status-test
+  - image: quay.io/redhat/olmtests:v0.0.1
+    entrypoint: 
+    - scorecard-test
+    - olm-bundle-validation
+    labels:
+      suite: olm
+      test: olm-bundle-validation-test
+  - image: quay.io/redhat/olmtests:v0.0.1
+    entrypoint: 
+    - scorecard-test
+    - olm-crds-have-validation
+    labels:
+      suite: olm
+      test: olm-crds-have-validation-test
+  - image: quay.io/redhat/kuttltests:v0.0.1
+    labels:
+      suite: kuttl
+    entrypoint:
+    - kuttl-test
+    - olm-status-descriptors
+`

internal/scorecard/alpha/run_test.go

@@ -16,6 +16,7 @@ package alpha
 
 import (
 	"context"
+	"errors"
 	"path/filepath"
 	"testing"
 	"time"
@@ -25,8 +26,8 @@ import (
 	"github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3"
 )
 
+// TODO(joelanford): rewrite to use ginkgo/gomega
 func TestRunTests(t *testing.T) {
-
 	cases := []struct {
 		name            string
 		configPathValue string
@@ -100,3 +101,92 @@ func TestRunTests(t *testing.T) {
 
 	}
 }
+
+// TODO(joelanford): rewrite to use ginkgo/gomega
+func TestRunParallelPass(t *testing.T) {
+	scorecard := getFakeScorecard(true)
+	ctx, cancel := context.WithTimeout(context.Background(), 7*time.Millisecond)
+	defer cancel()
+
+	tests, err := scorecard.Run(ctx)
+	if err != nil {
+		t.Fatalf("Expected no error, got error: %v", err)
+	}
+	if len(tests.Items) != 2 {
+		t.Fatalf("Expected 2 tests, got %d", len(tests.Items))
+	}
+	for _, test := range tests.Items {
+		expectPass(t, test)
+	}
+}
+
+// TODO(joelanford): rewrite to use ginkgo/gomega
+func TestRunSequentialPass(t *testing.T) {
+	scorecard := getFakeScorecard(false)
+	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Millisecond)
+	defer cancel()
+
+	tests, err := scorecard.Run(ctx)
+	if err != nil {
+		t.Fatalf("Expected no error, got error: %v", err)
+	}
+	if len(tests.Items) != 2 {
+		t.Fatalf("Expected 2 tests, got %d", len(tests.Items))
+	}
+	for _, test := range tests.Items {
+		expectPass(t, test)
+	}
+}
+
+// TODO(joelanford): rewrite to use ginkgo/gomega
+func TestRunSequentialFail(t *testing.T) {
+	scorecard := getFakeScorecard(false)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 7*time.Millisecond)
+	defer cancel()
+
+	_, err := scorecard.Run(ctx)
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("Expected deadline exceeded error, got:  %v", err)
+	}
+}
+
+func getFakeScorecard(parallel bool) Scorecard {
+	return Scorecard{
+		Config: Config{
+			Stages: []Stage{
+				{
+					Parallel: parallel,
+					Tests: []Test{
+						{},
+						{},
+					},
+				},
+			},
+		},
+		TestRunner: FakeTestRunner{
+			Sleep: 5 * time.Millisecond,
+			TestStatus: &v1alpha3.TestStatus{
+				Results: []v1alpha3.TestResult{
+					{
+						State: v1alpha3.PassState,
+					},
+				},
+			},
+		},
+	}
+}
+
+func expectPass(t *testing.T, test v1alpha3.Test) {
+	if len(test.Status.Results) != 1 {
+		t.Fatalf("Expected 1 results, got %d", len(test.Status.Results))
+	}
+	for _, r := range test.Status.Results {
+		if len(r.Errors) > 0 {
+			t.Fatalf("Expected no errors, got %v", r.Errors)
+		}
+		if r.State != v1alpha3.PassState {
+			t.Fatalf("Expected result state %q, got %q", v1alpha3.PassState, r.State)
+		}
+	}
+}