diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..3ba13e0ce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 000000000..f8e71f49e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,40 @@ +--- +name: Documentation 📝 +about: Suggest better docs coverage for a particular tool or process. +labels: "documentation" +--- + + + +## Summary + +What problem(s) did you run into that caused you to request additional documentation? What questions do you think we should answer? What, if any, existing documentation relates to this proposal? + +Some recommended topics to cover: + +- List the topics you think should be here. +- This list does not need to be exhaustive! + +### Motivation + +Why should we document this and who will benefit from it? + +## Steps to resolve this issue + + + +### Draft the doc + +- [ ] Write the doc, following the format listed in these resources: + - [Example of a similar article]() + +### Open a pull request + +- [ ] Open a pull request with your work including the words "closes #[this issue's number]" in the pull request description \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_suggestion.md b/.github/ISSUE_TEMPLATE/feature_suggestion.md new file mode 100644 index 000000000..1052b4a91 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_suggestion.md @@ -0,0 +1,26 @@ +--- +name: Feature Suggestion 💡 +about: Suggest a new idea for the project. +labels: "enhancement" +--- + + + +## Summary + +Brief explanation of the feature. + +### Basic example + +If the proposal involves a new or changed API, include a basic code example. Omit this section if it's not applicable. + +### Motivation + +Why are we doing this? What use cases does it support? What is the expected outcome? diff --git a/.github/ISSUE_TEMPLATE/plugin_suggestion.md b/.github/ISSUE_TEMPLATE/plugin_suggestion.md new file mode 100644 index 000000000..3042e3020 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/plugin_suggestion.md @@ -0,0 +1,26 @@ +--- +name: Plugin Suggestion 📦 +about: Suggest a new idea for the project. +labels: "new-plugin" +--- + + + +## Summary + +Brief explanation of the plugin. + +### Basic example + +If you have a POC, blog post or any other useful references please let us know in this section. + +### Motivation + +Why are we doing this? What use cases does it support? What is the expected outcome? diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 000000000..fc2194696 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,20 @@ +--- +name: Question 🤔 +about: Usage question or discussion about garak. +labels: "question" +--- + + + +## Summary + +## Relevant information + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..6876c14ff --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,34 @@ +Tell us what this change does. If you're fixing a bug, please mention +the github issue number. + +Please ensure you are submitting **from a unique branch** in your repository to `main` upstream. + +## Verification + +List the steps needed to make sure this thing works + +- [ ] Supporting configuration such as generator configuration file +``` json +{ + "huggingface": { + "torch_type": "float32" + } +} +``` +- [ ] `python -m -n ` +- [ ] ... +- [ ] **Verify** the thing does what it should +- [ ] **Verify** the thing does not do what it should not +- [ ] **Document** the thing and how it works ([Example](https://github.com/leondz/garak/blob/61ce5c4ae3caac08e0abd1d069d223d8a66104bd/garak/generators/rest.py#L24-L100)) + +If you are opening a PR for a new plugin that targets a **specific** piece of hardware or requires a **complex or hard-to-find** testing environment, we recommend that you send us as much detail as possible. + +Specific Hardware Examples: +* GPU related + * Specific support required `cuda` / `mps` ( Please not `cuda` via `ROCm` if related ) + * Minium GPU Memory + +Complex Software Examples: +* Expensive proprietary software +* Software with an extensive installation process +* Software without an English language UI diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml new file mode 100644 index 000000000..519721159 --- /dev/null +++ b/.github/workflows/labels.yml @@ -0,0 +1,164 @@ +name: Labels + +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions +permissions: + actions: none + checks: none + contents: none + deployments: none + id-token: none + # This action can update/close issues + issues: write + discussions: none + packages: none + pages: none + # This action can update/close pull requests + pull-requests: write + repository-projects: none + security-events: none + statuses: none + +on: + pull_request_target: + types: [labeled] + issues: + types: [labeled] + +jobs: + handle-labels: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v6 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + // NOTE: The following section is JavaScript. Note that backticks will need to be escaped within + // the multiline comment strings in the following config. When editing this file, using JavaScript + // syntax highlighting might be easier. + // + // This script has intentionally been inlined instead of using third-party Github actions for both + // security and performance reasons. + const allConfig = { + pullRequests: { + 'needs-linting': { + close: false, + comment: ` + Thanks for your pull request! Before this pull request can be merged, it must pass the checks of our automated linting tools. + + We use `black` to ensure the consistent format of our code. This can be run from the root directory of garak: + + \`\`\` + black garak + \`\`\` + + Please update your branch after these have been made, and reach out if you have any problems. + ` + }, + 'needs-unique-branch': { + close: true, + comment: ` + Thanks for your pull request! We require for all contributed code to come from a **from a unique branch** in your repository before it can be merged. + + Please create a new branch in your fork of garak and resubmit this from that branch. + + If you are using Git on the command line that may look like: + + \`\`\` + # Checkout the main branch + git checkout main + + # Create a new branch for your feature + git checkout -b + + # Add your new files + git add plugins/my-cool-new-plugin + + # Commit your changes with a relevant message + git commit + + # Push your changes to GitHub + git push origin + + # Now browse to the following URL and create your pull request! + # - https://github.com/leondz/garak/pulls + \`\`\` + + This helps protect the process, ensure users are aware of commits on the branch being considered for merge, allows for a location for more commits to be offered without mingling with other contributor changes and allows contributors to make progress while a PR is still being reviewed. + + Please do resubmit from a unique branch, we greatly value your contribution! :tada: + ` + }, + 'needs-testing-environment': { + close: false, + comment: ` + Thanks for your pull request! As part of our landing process, we manually verify that all plugins work as expected. + + We have been unable to test this plugin successfully. This may be due to software or hardware requirements we cannot replicate. + + To help unblock this pull request, please: + + - Comment with links to documentation on how to set up an environment, and provide exact software version numbers to use + - Or comment guided steps on how to set up our environment for testing this plugin + + Once there's a clear path for testing and evaluating this code, we can progress with this further. + ` + }, + 'needs-pull-request-template': { + close: false, + comment: ` + When creating a pull request, please ensure that the default pull request template has been updated with the required details. + ` + }, + }, + issues: { + 'needs-more-information': { + close: false, + comment: ` + It looks like there's not enough information to replicate this issue. Please provide any relevant output and logs which may be useful in diagnosing the issue. + + This includes: + + - All of the item points within this [template](https://github.com/leondz/garak/blob/master/.github/ISSUE_TEMPLATE/bug_report.md) + - Screenshots showing the issues you're having + - Exact replication steps + + The easier it is for us to replicate and debug an issue means there's a higher chance of this issue being resolved. + ` + }, + // Used for issues that have zero effort applied, potentially bot related + potato: { + close: true, + comment: ` + When creating an issue, please ensure that the default issue template has been updated with the required details: + https://github.com/leondz/garak/issues/new/choose + + Closing this issue. If you believe this issue has been closed in error, please provide any relevant output and logs which may be useful in diagnosing the issue. + ` + }, + } + }; + + const issueType = context.eventName === 'issues' ? 'issues' : 'pullRequests'; + const config = allConfig[issueType][context.payload.label.name]; + if (!config) { + return; + } + + if (config.comment) { + const precedingWhitespaceLength = config.comment.split("\n")[1].search(/\S/); + const commentWithoutPrecedingWhitespace = config.comment.split("\n").map(line => line.substring(precedingWhitespaceLength)).join("\n").trim(); + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: commentWithoutPrecedingWhitespace + }); + } + if (config.close) { + await github.rest.issues.update({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + state: 'closed' + }); + } \ No newline at end of file