diff --git a/backend/plugins/checkmarxone/README.md b/backend/plugins/checkmarxone/README.md new file mode 100644 index 00000000000..e2f76047d08 --- /dev/null +++ b/backend/plugins/checkmarxone/README.md @@ -0,0 +1,91 @@ +# CheckmarxOne Plugin + +## Summary + +This plugin collects security findings and vulnerabilities from [CheckmarxOne](https://checkmarx.com/checkmarx-one/) - a leading application security testing platform. + +## Features + +- Collect security findings/vulnerabilities from CheckmarxOne projects +- Track vulnerability severity, status, and remediation progress +- Support for multiple projects +- Integration with DevLake's security domain layer + +## Requirements + +- CheckmarxOne account and API access +- Server URL, Client ID, and Client Secret from CheckmarxOne + +## Configuration + +### Connection Setup + +Create a connection to CheckmarxOne using the following fields: + +- **Server URL**: The base URL of your CheckmarxOne instance (e.g., `https://checkmarx.mycompany.com`) +- **Client ID**: OAuth client ID for API access +- **Client Secret**: OAuth client secret for API access +- **Username**: (Optional) Username for authentication +- **Password**: (Optional) Password for authentication + +### Scope Configuration + +Select the CheckmarxOne projects you want to collect data from: + +- **Project ID**: The unique identifier of the CheckmarxOne project + +## Data Collection + +The plugin collects the following data: + +### Findings +- Finding ID and Name +- Severity Level (Critical, High, Medium, Low) +- Status (Open, Fixed, Suppressed) +- First Found and Last Found timestamps +- Finding Description +- Type of finding + +## API Reference + +### POST /connections +Create a new CheckmarxOne connection + +### GET /connections +List all CheckmarxOne connections + +### GET /connections/:connectionId +Get details of a specific connection + +### PATCH /connections/:connectionId +Update a CheckmarxOne connection + +### DELETE /connections/:connectionId +Delete a CheckmarxOne connection + +## Troubleshooting + +### Authentication Issues +- Verify Client ID and Client Secret are correct +- Ensure the API user has appropriate permissions in CheckmarxOne +- Check that the Server URL is accessible from the DevLake instance + +### No Data Collected +- Verify that the project ID exists in CheckmarxOne +- Check that the API client has access to the specified project +- Review the logs for any API errors + +## Development + +To build and test the plugin locally: + +```bash +cd backend/plugins/checkmarxone +go build +``` + +For standalone debugging: + +```bash +./checkmarxone --connectionId=1 --projectId=myproject +``` diff --git a/backend/plugins/checkmarxone/api/connection_api.go b/backend/plugins/checkmarxone/api/connection_api.go new file mode 100644 index 00000000000..ea5ace896d8 --- /dev/null +++ b/backend/plugins/checkmarxone/api/connection_api.go @@ -0,0 +1,109 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/http" + "strconv" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/checkmarxone/models" +) + +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.CheckmarxoneConnection{} + err := input.GetBody(connection) + if err != nil { + return nil, errors.BadInput.Wrap(err, "invalid request body") + } + + basicRes := input.Ctx.Value(plugin.CTX_KEY_BASIC_RES).(plugin.BasicRes) + if err := basicRes.GetDal().Create(connection); err != nil { + return nil, errors.Default.Wrap(err, "failed to create connection") + } + + return &plugin.ApiResourceOutput{Body: connection, Status: http.StatusCreated}, nil +} + +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connections []models.CheckmarxoneConnection + basicRes := input.Ctx.Value(plugin.CTX_KEY_BASIC_RES).(plugin.BasicRes) + + if err := basicRes.GetDal().All(&connections); err != nil { + return nil, errors.Default.Wrap(err, "failed to list connections") + } + + return &plugin.ApiResourceOutput{Body: connections}, nil +} + +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connectionId := input.Params["connectionId"] + connId, err := strconv.ParseUint(connectionId, 10, 64) + if err != nil { + return nil, errors.BadInput.New("invalid connection id") + } + + connection := &models.CheckmarxoneConnection{} + basicRes := input.Ctx.Value(plugin.CTX_KEY_BASIC_RES).(plugin.BasicRes) + + if err := basicRes.GetDal().First(connection, map[string]interface{}{"id": connId}); err != nil { + return nil, errors.NotFound.Wrap(err, "connection not found") + } + + return &plugin.ApiResourceOutput{Body: connection}, nil +} + +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connectionId := input.Params["connectionId"] + connId, err := strconv.ParseUint(connectionId, 10, 64) + if err != nil { + return nil, errors.BadInput.New("invalid connection id") + } + + connection := &models.CheckmarxoneConnection{} + basicRes := input.Ctx.Value(plugin.CTX_KEY_BASIC_RES).(plugin.BasicRes) + + err2 := input.GetBody(connection) + if err2 != nil { + return nil, errors.BadInput.Wrap(err2, "invalid request body") + } + + connection.ID = connId + if err := basicRes.GetDal().Update(connection); err != nil { + return nil, errors.Default.Wrap(err, "failed to update connection") + } + + return &plugin.ApiResourceOutput{Body: connection}, nil +} + +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connectionId := input.Params["connectionId"] + connId, err := strconv.ParseUint(connectionId, 10, 64) + if err != nil { + return nil, errors.BadInput.New("invalid connection id") + } + + basicRes := input.Ctx.Value(plugin.CTX_KEY_BASIC_RES).(plugin.BasicRes) + connection := &models.CheckmarxoneConnection{} + + if err := basicRes.GetDal().Delete(connection, map[string]interface{}{"id": connId}); err != nil { + return nil, errors.Default.Wrap(err, "failed to delete connection") + } + + return &plugin.ApiResourceOutput{Status: http.StatusNoContent}, nil +} diff --git a/backend/plugins/checkmarxone/api/init.go b/backend/plugins/checkmarxone/api/init.go new file mode 100644 index 00000000000..9632c62e56f --- /dev/null +++ b/backend/plugins/checkmarxone/api/init.go @@ -0,0 +1,19 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + diff --git a/backend/plugins/checkmarxone/checkmarxone.go b/backend/plugins/checkmarxone/checkmarxone.go new file mode 100644 index 00000000000..eb506eab9eb --- /dev/null +++ b/backend/plugins/checkmarxone/checkmarxone.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/checkmarxone/impl" + "github.com/spf13/cobra" +) + +// PluginEntry is a variable exported for Framework to search and load +var PluginEntry impl.CheckmarxOne //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "checkmarxone"} + connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "checkmarxone connection id") + projectId := cmd.Flags().StringP("projectId", "p", "", "checkmarxone project id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("connectionId") + _ = cmd.MarkFlagRequired("projectId") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "projectId": *projectId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/checkmarxone/impl/impl.go b/backend/plugins/checkmarxone/impl/impl.go new file mode 100644 index 00000000000..34fd04b8281 --- /dev/null +++ b/backend/plugins/checkmarxone/impl/impl.go @@ -0,0 +1,168 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/checkmarxone/models" + "github.com/apache/incubator-devlake/plugins/checkmarxone/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/checkmarxone/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginMigration + plugin.PluginSource + plugin.DataSourcePluginBlueprintV200 + plugin.CloseablePluginTask +} = (*CheckmarxOne)(nil) + +type CheckmarxOne struct{} + +func (p CheckmarxOne) Name() string { + return "checkmarxone" +} + +func (p CheckmarxOne) Description() string { + return "collect security findings from CheckmarxOne" +} + +func (p CheckmarxOne) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/checkmarxone" +} + +func (p CheckmarxOne) Init(basicRes context.BasicRes) errors.Error { + return nil +} + +func (p CheckmarxOne) Connection() dal.Tabler { + return &models.CheckmarxoneConnection{} +} + +func (p CheckmarxOne) Scope() plugin.ToolLayerScope { + return &models.CheckmarxoneProject{} +} + +func (p CheckmarxOne) ScopeConfig() dal.Tabler { + return nil +} + +func (p CheckmarxOne) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.CheckmarxoneConnection{}, + &models.CheckmarxoneProject{}, + &models.CheckmarxoneFinding{}, + } +} + +func (p CheckmarxOne) SubTaskMetas() []plugin.SubTaskMeta { + return tasks.CollectDataTaskMetas() +} + +func (p CheckmarxOne) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.CheckmarxoneOptions + err := plugin.Helper.JsonToModel(options, &op) + if err != nil { + return nil, errors.BadInput.Wrap(err, "invalid options") + } + + basicRes := taskCtx.GetContext().Value(plugin.CTX_KEY_BASIC_RES).(plugin.BasicRes) + connection := &models.CheckmarxoneConnection{} + err = basicRes.GetDal().First(connection, map[string]interface{}{"id": op.ConnectionId}) + if err != nil { + return nil, errors.NotFound.Wrap(err, "connection not found") + } + + logger := taskCtx.GetLogger() + apiClient, err := tasks.NewCheckmarxoneApiClient(logger, connection) + if err != nil { + return nil, err + } + + return &tasks.CheckmarxoneTaskData{ + Options: &op, + ApiClient: apiClient, + Connection: connection, + }, nil +} + +func (p CheckmarxOne) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.MigrationScripts{}.MigrationScripts() +} + +func (p CheckmarxOne) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "GET": api.GetConnection, + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + }, + } +} + +func (p CheckmarxOne) Close(taskCtx plugin.TaskContext) errors.Error { + data, _ := taskCtx.GetData().(*tasks.CheckmarxoneTaskData) + if data != nil && data.ApiClient != nil { + data.ApiClient.Close() + } + return nil +} + +func (p CheckmarxOne) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []plugin.Scope, + syncPolicy plugin.BlueprintSyncPolicy, +) (plugin.PipelinePlan, errors.Error) { + var err errors.Error + plan := plugin.PipelinePlan{} + + for _, scope := range scopes { + scopeItem, ok := scope.(*models.CheckmarxoneProject) + if !ok { + return nil, errors.BadInput.New("invalid scope item") + } + + stage := plugin.PipelineStage{} + for _, task := range p.SubTaskMetas() { + options := map[string]interface{}{ + "connectionId": connectionId, + "projectId": scopeItem.ProjectId, + } + stage = append(stage, &plugin.PipelineTask{ + Plugin: p.Name(), + Subtasks: []string{task.Name}, + Options: options, + }) + } + + plan = append(plan, stage) + } + + return plan, err +} diff --git a/backend/plugins/checkmarxone/models/connection.go b/backend/plugins/checkmarxone/models/connection.go new file mode 100644 index 00000000000..3a7d1b9d003 --- /dev/null +++ b/backend/plugins/checkmarxone/models/connection.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models" +) + +type CheckmarxoneConnection struct { + models.BaseConnection `mapstructure:",squash"` + ServerUrl string `mapstructure:"serverUrl" json:"serverUrl"` + Username string `mapstructure:"username" json:"username"` + Password string `mapstructure:"password" json:"-"` + ClientId string `mapstructure:"clientId" json:"clientId"` + ClientSecret string `mapstructure:"clientSecret" json:"-"` +} + +func (CheckmarxoneConnection) TableName() string { + return "checkmarxone_connections" +} diff --git a/backend/plugins/checkmarxone/models/finding.go b/backend/plugins/checkmarxone/models/finding.go new file mode 100644 index 00000000000..b4fe1c36f3b --- /dev/null +++ b/backend/plugins/checkmarxone/models/finding.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models" + "time" +) + +type CheckmarxoneFinding struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ProjectId string `gorm:"primaryKey" json:"projectId"` + FindingId string `gorm:"primaryKey" json:"findingId"` + Name string `json:"name"` + Severity string `json:"severity"` + Status string `json:"status"` + Description string `json:"description"` + FirstFound time.Time `json:"firstFound"` + LastFound time.Time `json:"lastFound"` + State string `json:"state"` + Type string `json:"type"` + Count int `json:"count"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + + models.NoPKModel +} + +func (CheckmarxoneFinding) TableName() string { + return "checkmarxone_findings" +} diff --git a/backend/plugins/checkmarxone/models/migrationscripts/register.go b/backend/plugins/checkmarxone/models/migrationscripts/register.go new file mode 100644 index 00000000000..679c91d05f4 --- /dev/null +++ b/backend/plugins/checkmarxone/models/migrationscripts/register.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/checkmarxone/models" +) + +var _ migrationscripts.MigrationScript = (*InitSchemas)(nil) + +type InitSchemas struct{} + +func (InitSchemas) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + err := db.AutoMigrate( + &models.CheckmarxoneConnection{}, + &models.CheckmarxoneProject{}, + &models.CheckmarxoneFinding{}, + ) + return err +} + +func (InitSchemas) Version() string { + return "20250101_init_schema" +} + +func (InitSchemas) Name() string { + return "init_schema" +} + +type MigrationScripts struct{} + +func (MigrationScripts) MigrationScripts() []plugin.MigrationScript { + return []plugin.MigrationScript{ + InitSchemas{}, + } +} diff --git a/backend/plugins/checkmarxone/models/project.go b/backend/plugins/checkmarxone/models/project.go new file mode 100644 index 00000000000..42fea176b3c --- /dev/null +++ b/backend/plugins/checkmarxone/models/project.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models" + "time" +) + +type CheckmarxoneProject struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ProjectId string `gorm:"primaryKey" json:"projectId"` + Name string `json:"name"` + Description string `json:"description"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + + models.NoPKModel +} + +func (CheckmarxoneProject) TableName() string { + return "checkmarxone_projects" +} diff --git a/backend/plugins/checkmarxone/tasks/api_client.go b/backend/plugins/checkmarxone/tasks/api_client.go new file mode 100644 index 00000000000..11e3d133f9f --- /dev/null +++ b/backend/plugins/checkmarxone/tasks/api_client.go @@ -0,0 +1,160 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/base64" + "fmt" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/utils" + "github.com/apache/incubator-devlake/plugins/checkmarxone/models" + "net/http" + "time" +) + +type CheckmarxoneApiClient struct { + client *http.Client + headers map[string]string + logger log.Logger + serverUrl string + username string + password string + clientId string + clientSecret string + token string + tokenExpire time.Time +} + +func NewCheckmarxoneApiClient(logger log.Logger, connection *models.CheckmarxoneConnection) (*CheckmarxoneApiClient, errors.Error) { + client := &CheckmarxoneApiClient{ + client: &http.Client{Timeout: 30 * time.Second}, + logger: logger, + serverUrl: connection.ServerUrl, + username: connection.Username, + password: connection.Password, + clientId: connection.ClientId, + clientSecret: connection.ClientSecret, + headers: map[string]string{ + "Accept": "application/json", + }, + } + + err := client.authenticate() + if err != nil { + return nil, err + } + + return client, nil +} + +func (c *CheckmarxoneApiClient) authenticate() errors.Error { + auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", c.clientId, c.clientSecret))) + + headers := map[string]string{ + "Authorization": fmt.Sprintf("Basic %s", auth), + "Content-Type": "application/x-www-form-urlencoded", + } + + res, err := utils.HTTPRequest( + "POST", + fmt.Sprintf("%s/auth/oauth/token", c.serverUrl), + nil, + map[string]string{"grant_type": "client_credentials"}, + headers, + c.client, + false, + ) + if err != nil { + return errors.Default.Wrap(err, "failed to authenticate with CheckmarxOne") + } + + if res.StatusCode != 200 { + return errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("failed to authenticate: %s", res.Body)) + } + + type TokenResponse struct { + AccessToken string `json:"access_token"` + ExpiresIn int `json:"expires_in"` + } + + var tokenResp TokenResponse + err = utils.UnmarshalResponse(res, &tokenResp) + if err != nil { + return errors.Default.Wrap(err, "failed to parse token response") + } + + c.token = tokenResp.AccessToken + c.tokenExpire = time.Now().Add(time.Duration(tokenResp.ExpiresIn) * time.Second) + c.headers["Authorization"] = fmt.Sprintf("Bearer %s", c.token) + + return nil +} + +func (c *CheckmarxoneApiClient) GetProjects() ([]map[string]interface{}, errors.Error) { + url := fmt.Sprintf("%s/api/projects", c.serverUrl) + return c.fetch(url) +} + +func (c *CheckmarxoneApiClient) GetFindings(projectId string) ([]map[string]interface{}, errors.Error) { + url := fmt.Sprintf("%s/api/projects/%s/results-summary", c.serverUrl, projectId) + return c.fetch(url) +} + +func (c *CheckmarxoneApiClient) fetch(url string) ([]map[string]interface{}, errors.Error) { + err := c.checkAndRefreshToken() + if err != nil { + return nil, err + } + + res, err := utils.HTTPRequest("GET", url, nil, nil, c.headers, c.client, false) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to fetch data from CheckmarxOne") + } + + if res.StatusCode != 200 { + return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("failed to fetch: %s", res.Body)) + } + + type DataResponse struct { + Results []map[string]interface{} `json:"results"` + Data []map[string]interface{} `json:"data"` + } + + var dataResp DataResponse + err = utils.UnmarshalResponse(res, &dataResp) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to parse response") + } + + if len(dataResp.Results) > 0 { + return dataResp.Results, nil + } + return dataResp.Data, nil +} + +func (c *CheckmarxoneApiClient) checkAndRefreshToken() errors.Error { + if time.Now().Before(c.tokenExpire) { + return nil + } + return c.authenticate() +} + +func (c *CheckmarxoneApiClient) Close() { + c.client.CloseIdleConnections() +} diff --git a/backend/plugins/checkmarxone/tasks/findings_collector.go b/backend/plugins/checkmarxone/tasks/findings_collector.go new file mode 100644 index 00000000000..5bc018f5356 --- /dev/null +++ b/backend/plugins/checkmarxone/tasks/findings_collector.go @@ -0,0 +1,60 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +const RAW_FINDINGS_TABLE = "checkmarxone_api_findings" + +var CollectFindingsMeta = plugin.SubTaskMeta{ + Name: "collectFindings", + EntryPoint: CollectFindings, + EnabledByDefault: true, + Description: "Collect findings from CheckmarxOne API", + DomainTypes: []string{plugin.DOMAIN_TYPE_SECURITY}, +} + +func CollectFindings(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*CheckmarxoneTaskData) + logger := taskCtx.GetLogger() + + findings, err := data.ApiClient.GetFindings(data.Options.ProjectId) + if err != nil { + logger.Error(err, "failed to fetch findings") + return err + } + + for _, finding := range findings { + select { + case <-taskCtx.GetContext().Done(): + return taskCtx.GetContext().Err() + default: + } + + err := taskCtx.SaveRawData(RAW_FINDINGS_TABLE, finding) + if err != nil { + logger.Error(err, "failed to save raw data") + return err + } + } + + return nil +} \ No newline at end of file diff --git a/backend/plugins/checkmarxone/tasks/findings_extractor.go b/backend/plugins/checkmarxone/tasks/findings_extractor.go new file mode 100644 index 00000000000..612a9b778ce --- /dev/null +++ b/backend/plugins/checkmarxone/tasks/findings_extractor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + checkmarxoneModels "github.com/apache/incubator-devlake/plugins/checkmarxone/models" +) + +var ExtractFindingsMeta = plugin.SubTaskMeta{ + Name: "extractFindings", + EntryPoint: ExtractFindings, + EnabledByDefault: true, + Description: "Extract findings data", + DomainTypes: []string{plugin.DOMAIN_TYPE_SECURITY}, +} + +var ExtractFindingsMeta = plugin.SubTaskMeta{ + Name: "extractFindings", + EntryPoint: ExtractFindings, + EnabledByDefault: true, + Description: "Extract findings data", + DomainTypes: []string{plugin.DOMAIN_TYPE_SECURITY}, +} + +func ExtractFindings(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*CheckmarxoneTaskData) + logger := taskCtx.GetLogger() + + extractor, err := plugin.NewDataConverter(plugin.DataConverterArgs{ + RawDataSubTaskArgs: plugin.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: RAW_FINDINGS_TABLE, + }, + InputRowType: func() interface{} { + return make(map[string]interface{}) + }, + Input: nil, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + rawData := inputRow.(map[string]interface{}) + + finding := checkmarxoneModels.CheckmarxoneFinding{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + } + + if id, ok := rawData["id"].(string); ok { + finding.FindingId = id + } + if name, ok := rawData["name"].(string); ok { + finding.Name = name + } + if severity, ok := rawData["severity"].(string); ok { + finding.Severity = severity + } + if status, ok := rawData["status"].(string); ok { + finding.Status = status + } + if desc, ok := rawData["description"].(string); ok { + finding.Description = desc + } + if state, ok := rawData["state"].(string); ok { + finding.State = state + } + if fType, ok := rawData["type"].(string); ok { + finding.Type = fType + } + if count, ok := rawData["count"].(float64); ok { + finding.Count = int(count) + } + + return []interface{}{&finding}, nil + }, + }) + if err != nil { + logger.Error(err, "failed to create converter") + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/checkmarxone/tasks/register.go b/backend/plugins/checkmarxone/tasks/register.go new file mode 100644 index 00000000000..158126b9e58 --- /dev/null +++ b/backend/plugins/checkmarxone/tasks/register.go @@ -0,0 +1,31 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +var TaskMetas = []plugin.SubTaskMeta{ + CollectFindingsMeta, + ExtractFindingsMeta, +} + +func CollectDataTaskMetas() []plugin.SubTaskMeta { + return TaskMetas +} diff --git a/backend/plugins/checkmarxone/tasks/task_data.go b/backend/plugins/checkmarxone/tasks/task_data.go new file mode 100644 index 00000000000..f56a0bd31a5 --- /dev/null +++ b/backend/plugins/checkmarxone/tasks/task_data.go @@ -0,0 +1,33 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/plugins/checkmarxone/models" +) + +type CheckmarxoneTaskData struct { + Options *CheckmarxoneOptions + ApiClient *CheckmarxoneApiClient + Connection *models.CheckmarxoneConnection +} + +type CheckmarxoneOptions struct { + ConnectionId uint64 `json:"connectionId"` + ProjectId string `json:"projectId"` +} diff --git a/backend/plugins/dora/tasks/change_lead_time_calculator.go b/backend/plugins/dora/tasks/change_lead_time_calculator.go index 47749b1af3f..7bfa52a2c21 100644 --- a/backend/plugins/dora/tasks/change_lead_time_calculator.go +++ b/backend/plugins/dora/tasks/change_lead_time_calculator.go @@ -288,16 +288,17 @@ func batchFetchFirstReviews(projectName string, db dal.Dal) (map[string]*code.Pu func batchFetchDeployments(projectName string, db dal.Dal) (map[string]*devops.CicdDeploymentCommit, errors.Error) { var results []*deploymentCommitWithMergeSha - // Query finds the first deployment for each merge commit by using a window function - // to rank deployments by started_date, then filtering to keep only rank 1. + // Query finds the first deployment for each merge commit. + // Only deployments with a previous successful deployment can define a valid change + // range; the first deployment is a seed deployment and should not be linked to + // historical merge requests. err := db.All( &results, dal.Select("dc.*, cd.commit_sha as merge_sha"), dal.From("cicd_deployment_commits dc"), - dal.Join("LEFT JOIN cicd_deployment_commits p ON dc.prev_success_deployment_commit_id = p.id"), - dal.Join("INNER JOIN commits_diffs cd ON cd.new_commit_sha = dc.commit_sha AND cd.old_commit_sha = COALESCE(p.commit_sha, '')"), + dal.Join("INNER JOIN cicd_deployment_commits p ON dc.prev_success_deployment_commit_id = p.id"), + dal.Join("INNER JOIN commits_diffs cd ON cd.new_commit_sha = dc.commit_sha AND cd.old_commit_sha = p.commit_sha"), dal.Join("LEFT JOIN project_mapping pm ON pm.table = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id"), - dal.Where("dc.prev_success_deployment_commit_id <> ''"), dal.Where("dc.environment = 'PRODUCTION'"), // TODO: remove this when multi-environment is supported dal.Where("dc.result = ? AND pm.project_name = ?", devops.RESULT_SUCCESS, projectName), dal.Orderby("cd.commit_sha, dc.started_date ASC, dc.id ASC"),