Rename scripts to build and add revive command as a new build tool command (#10942)

Co-authored-by: techknowlogick <techknowlogick@gitea.io>

build/generate-bindata.go

@@ -0,0 +1,86 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/shurcooL/vfsgen"
+)
+
+func needsUpdate(dir string, filename string) (bool, []byte) {
+	needRegen := false
+	_, err := os.Stat(filename)
+	if err != nil {
+		needRegen = true
+	}
+
+	oldHash, err := ioutil.ReadFile(filename + ".hash")
+	if err != nil {
+		oldHash = []byte{}
+	}
+
+	hasher := sha1.New()
+
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		_, _ = hasher.Write([]byte(info.Name()))
+		_, _ = hasher.Write([]byte(info.ModTime().String()))
+		_, _ = hasher.Write([]byte(strconv.FormatInt(info.Size(), 16)))
+		return nil
+	})
+	if err != nil {
+		return true, oldHash
+	}
+
+	newHash := hasher.Sum([]byte{})
+
+	if bytes.Compare(oldHash, newHash) != 0 {
+
+		return true, newHash
+	}
+
+	return needRegen, newHash
+}
+
+func main() {
+	if len(os.Args) != 4 {
+		log.Fatal("Insufficient number of arguments. Need: directory packageName filename")
+	}
+
+	dir, packageName, filename := os.Args[1], os.Args[2], os.Args[3]
+
+	update, newHash := needsUpdate(dir, filename)
+
+	if !update {
+		fmt.Printf("bindata for %s already up-to-date\n", packageName)
+		return
+	}
+
+	fmt.Printf("generating bindata for %s\n", packageName)
+	var fsTemplates http.FileSystem = http.Dir(dir)
+	err := vfsgen.Generate(fsTemplates, vfsgen.Options{
+		PackageName:  packageName,
+		BuildTags:    "bindata",
+		VariableName: "Assets",
+		Filename:     filename,
+	})
+	if err != nil {
+		log.Fatalf("%v\n", err)
+	}
+	_ = ioutil.WriteFile(filename+".hash", newHash, 0666)
+}

build/generate-gitignores.go

@@ -0,0 +1,117 @@
+// +build ignore
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	var (
+		prefix      = "gitea-gitignore"
+		url         = "https://api.github.com/repos/github/gitignore/tarball"
+		destination = ""
+	)
+
+	flag.StringVar(&destination, "dest", "options/gitignore/", "destination for the gitignores")
+	flag.Parse()
+
+	file, err := ioutil.TempFile(os.TempDir(), prefix)
+
+	if err != nil {
+		log.Fatalf("Failed to create temp file. %s", err)
+	}
+
+	defer os.Remove(file.Name())
+
+	resp, err := http.Get(url)
+
+	if err != nil {
+		log.Fatalf("Failed to download archive. %s", err)
+	}
+
+	defer resp.Body.Close()
+
+	if _, err := io.Copy(file, resp.Body); err != nil {
+		log.Fatalf("Failed to copy archive to file. %s", err)
+	}
+
+	if _, err := file.Seek(0, 0); err != nil {
+		log.Fatalf("Failed to reset seek on archive. %s", err)
+	}
+
+	gz, err := gzip.NewReader(file)
+
+	if err != nil {
+		log.Fatalf("Failed to gunzip the archive. %s", err)
+	}
+
+	tr := tar.NewReader(gz)
+
+	filesToCopy := make(map[string]string, 0)
+
+	for {
+		hdr, err := tr.Next()
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			log.Fatalf("Failed to iterate archive. %s", err)
+		}
+
+		if filepath.Ext(hdr.Name) != ".gitignore" {
+			continue
+		}
+
+		if hdr.Typeflag == tar.TypeSymlink {
+			fmt.Printf("Found symlink %s -> %s\n", hdr.Name, hdr.Linkname)
+			filesToCopy[strings.TrimSuffix(filepath.Base(hdr.Name), ".gitignore")] = strings.TrimSuffix(filepath.Base(hdr.Linkname), ".gitignore")
+			continue
+		}
+
+		out, err := os.Create(path.Join(destination, strings.TrimSuffix(filepath.Base(hdr.Name), ".gitignore")))
+
+		if err != nil {
+			log.Fatalf("Failed to create new file. %s", err)
+		}
+
+		defer out.Close()
+
+		if _, err := io.Copy(out, tr); err != nil {
+			log.Fatalf("Failed to write new file. %s", err)
+		} else {
+			fmt.Printf("Written %s\n", out.Name())
+		}
+	}
+
+	for dst, src := range filesToCopy {
+		// Read all content of src to data
+		src = path.Join(destination, src)
+		data, err := ioutil.ReadFile(src)
+		if err != nil {
+			log.Fatalf("Failed to read src file. %s", err)
+		}
+		// Write data to dst
+		dst = path.Join(destination, dst)
+		err = ioutil.WriteFile(dst, data, 0644)
+		if err != nil {
+			log.Fatalf("Failed to write new file. %s", err)
+		}
+		fmt.Printf("Written (copy of %s) %s\n", src, dst)
+	}
+
+	fmt.Println("Done")
+}

build/generate-licenses.go

@@ -0,0 +1,104 @@
+// +build ignore
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	var (
+		prefix      = "gitea-licenses"
+		url         = "https://api.github.com/repos/spdx/license-list-data/tarball"
+		destination = ""
+	)
+
+	flag.StringVar(&destination, "dest", "options/license/", "destination for the licenses")
+	flag.Parse()
+
+	file, err := ioutil.TempFile(os.TempDir(), prefix)
+
+	if err != nil {
+		log.Fatalf("Failed to create temp file. %s", err)
+	}
+
+	defer os.Remove(file.Name())
+
+	resp, err := http.Get(url)
+
+	if err != nil {
+		log.Fatalf("Failed to download archive. %s", err)
+	}
+
+	defer resp.Body.Close()
+
+	if _, err := io.Copy(file, resp.Body); err != nil {
+		log.Fatalf("Failed to copy archive to file. %s", err)
+	}
+
+	if _, err := file.Seek(0, 0); err != nil {
+		log.Fatalf("Failed to reset seek on archive. %s", err)
+	}
+
+	gz, err := gzip.NewReader(file)
+
+	if err != nil {
+		log.Fatalf("Failed to gunzip the archive. %s", err)
+	}
+
+	tr := tar.NewReader(gz)
+
+	for {
+		hdr, err := tr.Next()
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			log.Fatalf("Failed to iterate archive. %s", err)
+		}
+
+		if !strings.Contains(hdr.Name, "/text/") {
+			continue
+		}
+
+		if filepath.Ext(hdr.Name) != ".txt" {
+			continue
+		}
+
+		if strings.HasPrefix(filepath.Base(hdr.Name), "README") {
+			continue
+		}
+
+		if strings.HasPrefix(filepath.Base(hdr.Name), "deprecated_") {
+			continue
+		}
+		out, err := os.Create(path.Join(destination, strings.TrimSuffix(filepath.Base(hdr.Name), ".txt")))
+
+		if err != nil {
+			log.Fatalf("Failed to create new file. %s", err)
+		}
+
+		defer out.Close()
+
+		if _, err := io.Copy(out, tr); err != nil {
+			log.Fatalf("Failed to write new file. %s", err)
+		} else {
+			fmt.Printf("Written %s\n", out.Name())
+		}
+	}
+
+	fmt.Println("Done")
+}

build/lint.go

@@ -0,0 +1,325 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Copyright (c) 2018 Minko Gechev. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"github.com/mgechev/dots"
+	"github.com/mgechev/revive/formatter"
+	"github.com/mgechev/revive/lint"
+	"github.com/mgechev/revive/rule"
+	"github.com/mitchellh/go-homedir"
+)
+
+func fail(err string) {
+	fmt.Fprintln(os.Stderr, err)
+	os.Exit(1)
+}
+
+var defaultRules = []lint.Rule{
+	&rule.VarDeclarationsRule{},
+	&rule.PackageCommentsRule{},
+	&rule.DotImportsRule{},
+	&rule.BlankImportsRule{},
+	&rule.ExportedRule{},
+	&rule.VarNamingRule{},
+	&rule.IndentErrorFlowRule{},
+	&rule.IfReturnRule{},
+	&rule.RangeRule{},
+	&rule.ErrorfRule{},
+	&rule.ErrorNamingRule{},
+	&rule.ErrorStringsRule{},
+	&rule.ReceiverNamingRule{},
+	&rule.IncrementDecrementRule{},
+	&rule.ErrorReturnRule{},
+	&rule.UnexportedReturnRule{},
+	&rule.TimeNamingRule{},
+	&rule.ContextKeysType{},
+	&rule.ContextAsArgumentRule{},
+}
+
+var allRules = append([]lint.Rule{
+	&rule.ArgumentsLimitRule{},
+	&rule.CyclomaticRule{},
+	&rule.FileHeaderRule{},
+	&rule.EmptyBlockRule{},
+	&rule.SuperfluousElseRule{},
+	&rule.ConfusingNamingRule{},
+	&rule.GetReturnRule{},
+	&rule.ModifiesParamRule{},
+	&rule.ConfusingResultsRule{},
+	&rule.DeepExitRule{},
+	&rule.UnusedParamRule{},
+	&rule.UnreachableCodeRule{},
+	&rule.AddConstantRule{},
+	&rule.FlagParamRule{},
+	&rule.UnnecessaryStmtRule{},
+	&rule.StructTagRule{},
+	&rule.ModifiesValRecRule{},
+	&rule.ConstantLogicalExprRule{},
+	&rule.BoolLiteralRule{},
+	&rule.RedefinesBuiltinIDRule{},
+	&rule.ImportsBlacklistRule{},
+	&rule.FunctionResultsLimitRule{},
+	&rule.MaxPublicStructsRule{},
+	&rule.RangeValInClosureRule{},
+	&rule.RangeValAddress{},
+	&rule.WaitGroupByValueRule{},
+	&rule.AtomicRule{},
+	&rule.EmptyLinesRule{},
+	&rule.LineLengthLimitRule{},
+	&rule.CallToGCRule{},
+	&rule.DuplicatedImportsRule{},
+	&rule.ImportShadowingRule{},
+	&rule.BareReturnRule{},
+	&rule.UnusedReceiverRule{},
+	&rule.UnhandledErrorRule{},
+	&rule.CognitiveComplexityRule{},
+	&rule.StringOfIntRule{},
+}, defaultRules...)
+
+var allFormatters = []lint.Formatter{
+	&formatter.Stylish{},
+	&formatter.Friendly{},
+	&formatter.JSON{},
+	&formatter.NDJSON{},
+	&formatter.Default{},
+	&formatter.Unix{},
+	&formatter.Checkstyle{},
+	&formatter.Plain{},
+}
+
+func getFormatters() map[string]lint.Formatter {
+	result := map[string]lint.Formatter{}
+	for _, f := range allFormatters {
+		result[f.Name()] = f
+	}
+	return result
+}
+
+func getLintingRules(config *lint.Config) []lint.Rule {
+	rulesMap := map[string]lint.Rule{}
+	for _, r := range allRules {
+		rulesMap[r.Name()] = r
+	}
+
+	lintingRules := []lint.Rule{}
+	for name := range config.Rules {
+		rule, ok := rulesMap[name]
+		if !ok {
+			fail("cannot find rule: " + name)
+		}
+		lintingRules = append(lintingRules, rule)
+	}
+
+	return lintingRules
+}
+
+func parseConfig(path string) *lint.Config {
+	config := &lint.Config{}
+	file, err := ioutil.ReadFile(path)
+	if err != nil {
+		fail("cannot read the config file")
+	}
+	_, err = toml.Decode(string(file), config)
+	if err != nil {
+		fail("cannot parse the config file: " + err.Error())
+	}
+	return config
+}
+
+func normalizeConfig(config *lint.Config) {
+	if config.Confidence == 0 {
+		config.Confidence = 0.8
+	}
+	severity := config.Severity
+	if severity != "" {
+		for k, v := range config.Rules {
+			if v.Severity == "" {
+				v.Severity = severity
+			}
+			config.Rules[k] = v
+		}
+		for k, v := range config.Directives {
+			if v.Severity == "" {
+				v.Severity = severity
+			}
+			config.Directives[k] = v
+		}
+	}
+}
+
+func getConfig() *lint.Config {
+	config := defaultConfig()
+	if configPath != "" {
+		config = parseConfig(configPath)
+	}
+	normalizeConfig(config)
+	return config
+}
+
+func getFormatter() lint.Formatter {
+	formatters := getFormatters()
+	formatter := formatters["default"]
+	if formatterName != "" {
+		f, ok := formatters[formatterName]
+		if !ok {
+			fail("unknown formatter " + formatterName)
+		}
+		formatter = f
+	}
+	return formatter
+}
+
+func buildDefaultConfigPath() string {
+	var result string
+	if homeDir, err := homedir.Dir(); err == nil {
+		result = filepath.Join(homeDir, "revive.toml")
+		if _, err := os.Stat(result); err != nil {
+			result = ""
+		}
+	}
+
+	return result
+}
+
+func defaultConfig() *lint.Config {
+	defaultConfig := lint.Config{
+		Confidence: 0.0,
+		Severity:   lint.SeverityWarning,
+		Rules:      map[string]lint.RuleConfig{},
+	}
+	for _, r := range defaultRules {
+		defaultConfig.Rules[r.Name()] = lint.RuleConfig{}
+	}
+	return &defaultConfig
+}
+
+func normalizeSplit(strs []string) []string {
+	res := []string{}
+	for _, s := range strs {
+		t := strings.Trim(s, " \t")
+		if len(t) > 0 {
+			res = append(res, t)
+		}
+	}
+	return res
+}
+
+func getPackages() [][]string {
+	globs := normalizeSplit(flag.Args())
+	if len(globs) == 0 {
+		globs = append(globs, ".")
+	}
+
+	packages, err := dots.ResolvePackages(globs, normalizeSplit(excludePaths))
+	if err != nil {
+		fail(err.Error())
+	}
+
+	return packages
+}
+
+type arrayFlags []string
+
+func (i *arrayFlags) String() string {
+	return strings.Join([]string(*i), " ")
+}
+
+func (i *arrayFlags) Set(value string) error {
+	*i = append(*i, value)
+	return nil
+}
+
+var configPath string
+var excludePaths arrayFlags
+var formatterName string
+var help bool
+
+var originalUsage = flag.Usage
+
+func init() {
+	flag.Usage = func() {
+		originalUsage()
+	}
+	// command line help strings
+	const (
+		configUsage    = "path to the configuration TOML file, defaults to $HOME/revive.toml, if present (i.e. -config myconf.toml)"
+		excludeUsage   = "list of globs which specify files to be excluded (i.e. -exclude foo/...)"
+		formatterUsage = "formatter to be used for the output (i.e. -formatter stylish)"
+	)
+
+	defaultConfigPath := buildDefaultConfigPath()
+
+	flag.StringVar(&configPath, "config", defaultConfigPath, configUsage)
+	flag.Var(&excludePaths, "exclude", excludeUsage)
+	flag.StringVar(&formatterName, "formatter", "", formatterUsage)
+	flag.Parse()
+}
+
+func main() {
+	config := getConfig()
+	formatter := getFormatter()
+	packages := getPackages()
+
+	revive := lint.New(func(file string) ([]byte, error) {
+		return ioutil.ReadFile(file)
+	})
+
+	lintingRules := getLintingRules(config)
+
+	failures, err := revive.Lint(packages, lintingRules, *config)
+	if err != nil {
+		fail(err.Error())
+	}
+
+	formatChan := make(chan lint.Failure)
+	exitChan := make(chan bool)
+
+	var output string
+	go (func() {
+		output, err = formatter.Format(formatChan, *config)
+		if err != nil {
+			fail(err.Error())
+		}
+		exitChan <- true
+	})()
+
+	exitCode := 0
+	for f := range failures {
+		if f.Confidence < config.Confidence {
+			continue
+		}
+		if exitCode == 0 {
+			exitCode = config.WarningCode
+		}
+		if c, ok := config.Rules[f.RuleName]; ok && c.Severity == lint.SeverityError {
+			exitCode = config.ErrorCode
+		}
+		if c, ok := config.Directives[f.RuleName]; ok && c.Severity == lint.SeverityError {
+			exitCode = config.ErrorCode
+		}
+
+		formatChan <- f
+	}
+
+	close(formatChan)
+	<-exitChan
+	if output != "" {
+		fmt.Println(output)
+	}
+
+	os.Exit(exitCode)
+}

build/update-locales.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+mv ./options/locale/locale_en-US.ini ./options/
+
+# Make sure to only change lines that have the translation enclosed between quotes
+sed -i -r -e '/^[a-zA-Z0-9_.-]+[ ]*=[ ]*".*"$/ {
+	s/^([a-zA-Z0-9_.-]+)[ ]*="/\1=/
+	s/\\"/"/g
+	s/"$//
+	}' ./options/locale/*.ini
+
+# Remove translation under 25% of en_us
+baselines=`wc -l "./options/locale_en-US.ini" | cut -d" " -f1`
+baselines=$((baselines / 4))
+for filename in ./options/locale/*.ini; do
+  lines=`wc -l "$filename" | cut -d" " -f1`
+  if [ $lines -lt $baselines ]; then
+    echo "Removing $filename: $lines/$baselines"
+    rm "$filename"
+  fi
+done
+
+mv ./options/locale_en-US.ini ./options/locale/

build/vendor.go

@@ -0,0 +1,18 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package build
+
+import (
+	// for lint
+	_ "github.com/BurntSushi/toml"
+	_ "github.com/mgechev/dots"
+	_ "github.com/mgechev/revive/formatter"
+	_ "github.com/mgechev/revive/lint"
+	_ "github.com/mgechev/revive/rule"
+	_ "github.com/mitchellh/go-homedir"
+
+	// for embed
+	_ "github.com/shurcooL/vfsgen"
+)