diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..c01af01d40 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +**/node_modules +mysql +data +data-* diff --git a/.drone.yml b/.drone.yml index 6406367db9..8d6041bf11 100644 --- a/.drone.yml +++ b/.drone.yml @@ -108,18 +108,18 @@ steps: - make frontend depends_on: [test-frontend] - - name: build-backend-no-gcc - image: golang:1.17 # this step is kept as the lowest version of golang that we support - pull: always - environment: - GO111MODULE: on - GOPROXY: https://goproxy.io - commands: - - go build -o gitea_no_gcc # test if build succeeds without the sqlite tag - depends_on: [deps-backend, checks-backend] - volumes: - - name: deps - path: /go + # - name: build-backend-no-gcc + # image: golang:1.17 # this step is kept as the lowest version of golang that we support + # pull: always + # environment: + # GO111MODULE: on + # GOPROXY: https://goproxy.io + # commands: + # - go build -o gitea_no_gcc # test if build succeeds without the sqlite tag + # depends_on: [deps-backend, checks-backend] + # volumes: + # - name: deps + # path: /go - name: build-backend-arm64 image: golang:1.18 diff --git a/Dockerfile-dev b/Dockerfile-dev new file mode 100644 index 0000000000..5442b88952 --- /dev/null +++ b/Dockerfile-dev @@ -0,0 +1,55 @@ +#Build stage +FROM golang:1.18-alpine3.16 + +ARG GOPROXY +ENV GOPROXY ${GOPROXY:-direct} + +ARG GITEA_VERSION +ARG TAGS="sqlite sqlite_unlock_notify sqlite_json" +ENV TAGS "bindata timetzdata $TAGS" +ARG CGO_EXTRA_CFLAGS + +#Setup repo +COPY . ${GOPATH}/src/code.gitea.io/gitea +WORKDIR ${GOPATH}/src/code.gitea.io/gitea + +EXPOSE 22 3000 80 + +RUN apk --no-cache add \ + bash \ + ca-certificates \ + curl \ + gettext \ + linux-pam \ + openssh \ + s6 \ + sqlite \ + su-exec \ + gnupg \ + make \ + build-base \ + git \ + nodejs \ + npm \ + build-base + +RUN apk add git --repository=http://dl-cdn.alpinelinux.org/alpine/v3.16/main + +RUN addgroup \ + -S -g 1000 \ + git && \ + adduser \ + -S -H -D \ + -h /data/git \ + -s /bin/bash \ + -u 1000 \ + -G git \ + git && \ + echo "git:*" | chpasswd -e + +ENV USER git +ENV GITEA_CUSTOM /data/gitea + +VOLUME ["/data"] + +CMD sh -c 'ln -s /go/src/code.gitea.io /app'; tail -f /data/gitea/log/gitea.log diff --git a/docker-compose.develop.yml b/docker-compose.develop.yml new file mode 100644 index 0000000000..38150a4746 --- /dev/null +++ b/docker-compose.develop.yml @@ -0,0 +1,52 @@ +version: "3" + +services: + dcs: + container_name: dcs + image: dcs-dev:latest + build: + context: . + dockerfile: Dockerfile-dev + user: "${UID}:${GID}" + environment: + - USER_UID="${UID}" + - USER_GID="${GID}" + - GITEA__database__DB_TYPE=mysql + - GITEA__database__HOST=db:3306 + - GITEA__database__NAME=gitea-release + - GITEA__database__USER=gitea + - GITEA__database__PASSWD=gitea + - TAGS=bindata sqlite sqlite_unlock_notify sqlite_json + restart: always + networks: + - gitea + volumes: + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + - ./data-release:/data + - .:/go/src/code.gitea.io/gitea + ports: + - "${DCS_PORT-3000}:80" + - "222:22" + depends_on: + - db + stdin_open: true # docker run -i + tty: true # docker run -t + + db: + platform: linux/x86_64 + image: mysql:5.7 + restart: always + environment: + - MYSQL_ROOT_PASSWORD=gitea + - MYSQL_USER=gitea + - MYSQL_PASSWORD=gitea + - MYSQL_DATABASE=gitea-release + networks: + - gitea + volumes: + - ./mysql:/var/lib/mysql + +networks: + gitea: + external: false diff --git a/docker-compose.main.yml b/docker-compose.main.yml index 18b68bb9db..7aedadaa33 100644 --- a/docker-compose.main.yml +++ b/docker-compose.main.yml @@ -2,6 +2,7 @@ version: "3" services: dcs: + container_name: dcs build: . environment: - USER_UID=1000 @@ -19,7 +20,7 @@ services: - /etc/localtime:/etc/localtime:ro - ./data-main:/data ports: - - "${DCS_PORT-3000}:3000" + - "${DCS_PORT-3000}:80" - "222:22" depends_on: - db diff --git a/docker-compose.release.yml b/docker-compose.release.yml index 3d840ead1a..013dc25f07 100644 --- a/docker-compose.release.yml +++ b/docker-compose.release.yml @@ -2,6 +2,7 @@ version: "3" services: dcs: + container_name: dcs build: . image: dcs-local:release environment: @@ -14,13 +15,13 @@ services: - GITEA__database__PASSWD=gitea restart: always networks: - - gitea + - gitea volumes: - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro - ./data-release:/data ports: - - "${DCS_PORT-3000}:3000" + - "${DCS_PORT-3000}:80" - "222:22" depends_on: - db diff --git a/models/door43metadata.go b/models/door43metadata.go index e70858f5ff..eb2020827c 100644 --- a/models/door43metadata.go +++ b/models/door43metadata.go @@ -75,7 +75,10 @@ func (dm *Door43Metadata) getRelease(e db.Engine) error { return err } dm.Release.Repo = dm.Repo - return dm.Release.LoadAttributes() + if err := dm.Release.LoadAttributes(); err != nil { + log.Warn("loadAttributes Error: %v\n", err) + return err + } } return nil } @@ -86,6 +89,7 @@ func (dm *Door43Metadata) loadAttributes(e db.Engine) error { } if dm.Release == nil && dm.ReleaseID > 0 { if err := dm.getRelease(e); err != nil { + log.Error("getRelease: %v", err) return nil } } @@ -188,7 +192,7 @@ func (dm *Door43Metadata) GetContentsURL() string { return fmt.Sprintf("%s/contents?ref=%s", dm.Repo.APIURL(), dm.BranchOrTag) } -// GetBooks get the books of the resource +// GetBooks get the books of the manifest func (dm *Door43Metadata) GetBooks() []string { var books []string if len((*dm.Metadata)["projects"].([]interface{})) > 0 { diff --git a/models/release.go b/models/release.go index f806ba2c51..09a7fd446c 100644 --- a/models/release.go +++ b/models/release.go @@ -152,7 +152,6 @@ func AddReleaseAttachments(ctx context.Context, releaseID int64, attachmentUUIDs return fmt.Errorf("update attachment [%d]: %v", attachments[i].ID, err) } } - return } diff --git a/models/repo/attachment.go b/models/repo/attachment.go index 3fb331a202..3a35a1389b 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -9,6 +9,7 @@ import ( "fmt" "net/url" "path" + "strings" // DCS Customizations "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/setting" @@ -29,12 +30,50 @@ type Attachment struct { DownloadCount int64 `xorm:"DEFAULT 0"` Size int64 `xorm:"DEFAULT 0"` CreatedUnix timeutil.TimeStamp `xorm:"created"` + /*** DCS Customizations ***/ + BrowserDownloadURL string `xorm:"-" json:"browser_download_url"` + /*** END DCS Customizations ***/ } func init() { db.RegisterModel(new(Attachment)) } +/*** DCS Customizations ***/ + +func (a *Attachment) AfterLoad() { + if strings.Contains(a.Name, "|http") || strings.Contains(a.Name, "|ftp") { + if name, url, ok := strings.Cut(a.Name, "|"); ok { + a.Name = name + a.BrowserDownloadURL = url + } + } +} + +func (a *Attachment) BeforeInsert() { + if a.Name == "" && a.BrowserDownloadURL != "" { + u, _ := url.Parse(a.BrowserDownloadURL) + a.Name = path.Base(u.Path) + } + if a.BrowserDownloadURL != "" { + a.Name = fmt.Sprintf("%s|%s", a.Name, a.BrowserDownloadURL) + } +} + +func (a *Attachment) BeforeUpdate() { + a.BeforeInsert() +} + +func (a *Attachment) AfterInsert() { + a.AfterLoad() +} + +func (a *Attachment) AfterUpdate() { + a.AfterLoad() +} + +/*** END DCS Customiations ***/ + // IncreaseDownloadCount is update download count + 1 func (a *Attachment) IncreaseDownloadCount() error { // Update download count. @@ -57,6 +96,11 @@ func (a *Attachment) RelativePath() string { // DownloadURL returns the download url of the attached file func (a *Attachment) DownloadURL() string { + /*** DCS Customizations ***/ + if a.BrowserDownloadURL != "" { + return a.BrowserDownloadURL + } + /*** END DCS Customizations ***/ return setting.AppURL + "attachments/" + url.PathEscape(a.UUID) } diff --git a/models/repo/repo.go b/models/repo/repo.go index f13b4359ae..bf3242c9a5 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -252,6 +252,20 @@ func (repo *Repository) APIURL() string { return setting.AppURL + "api/v1/repos/" + url.PathEscape(repo.OwnerName) + "/" + url.PathEscape(repo.Name) } +/*** DCS Customizations ***/ + +// CatalogSearchURL returns the repository catalog search API URL +func (repo *Repository) CatalogSearchURL() string { + return setting.AppURL + "api/catalog/latest/search/" + url.PathEscape(repo.OwnerName) + "/" + url.PathEscape(repo.Name) +} + +// CatalogEntryURL returns the repository catalog entry API URL +func (repo *Repository) CatalogEntryURL() string { + return setting.AppURL + "api/catalog/latest/entry/" + url.PathEscape(repo.OwnerName) + "/" + url.PathEscape(repo.Name) +} + +/*** END DCS Customizations ***/ + // GetCommitsCountCacheKey returns cache key used for commits count caching. func (repo *Repository) GetCommitsCountCacheKey(contextName string, isRef bool) string { var prefix string diff --git a/models/repo_list_test.go b/models/repo_list_test.go index 43b710a906..d1be525ef7 100644 --- a/models/repo_list_test.go +++ b/models/repo_list_test.go @@ -234,13 +234,13 @@ func TestSearchRepository(t *testing.T) { { name: "AllPublic/PublicRepositoriesOfUserIncludingCollaborative", opts: &SearchRepoOptions{ListOptions: db.ListOptions{Page: 1, PageSize: 10}, OwnerID: 15, AllPublic: true, Template: util.OptionalBoolFalse}, - /*** DCS Customization - Adds 1 more test, total 29 ***/ + /*** DCS Customizations - Adds 1 more test, total 29 ***/ count: 29, }, { name: "AllPublic/PublicAndPrivateRepositoriesOfUserIncludingCollaborative", opts: &SearchRepoOptions{ListOptions: db.ListOptions{Page: 1, PageSize: 10}, OwnerID: 15, Private: true, AllPublic: true, AllLimited: true, Template: util.OptionalBoolFalse}, - /*** DCS Customization - Adds 1 more test, total 34 ***/ + /*** DCS Customizations - Adds 1 more test, total 34 ***/ count: 34, }, { @@ -256,7 +256,7 @@ func TestSearchRepository(t *testing.T) { { name: "AllPublic/PublicRepositoriesOfOrganization", opts: &SearchRepoOptions{ListOptions: db.ListOptions{Page: 1, PageSize: 10}, OwnerID: 17, AllPublic: true, Collaborate: util.OptionalBoolFalse, Template: util.OptionalBoolFalse}, - /*** DCS Customization - Adds 1 more test, total 29 ***/ + /*** DCS Customizations - Adds 1 more test, total 29 ***/ count: 29, }, { diff --git a/modules/convert/catalog_next.go b/modules/convert/catalog_next.go index fd1bf2de95..fcce5720f7 100644 --- a/modules/convert/catalog_next.go +++ b/modules/convert/catalog_next.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/perm" + "code.gitea.io/gitea/modules/dcs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -114,6 +115,47 @@ func ToCatalogV5(dm *models.Door43Metadata, mode perm.AccessMode) *api.CatalogV5 release = ToRelease(dm.Release) } + var language string + if val, ok := (*dm.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["identifier"].(string); ok { + language = val + } + + var languageDir = "ltr" + if val, ok := (*dm.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["direction"].(string); ok { + languageDir = val + } else if language != "" { + dcs.GetLanguageDirection(language) + } + + var languageTitle string + if val, ok := (*dm.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["title"].(string); ok { + languageTitle = val + } else if language != "" { + dcs.GetLanguageTitle(language) + } + + var languageIsGL bool + if val, ok := (*dm.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["is_gl"].(bool); ok { + languageIsGL = val + } else { + languageIsGL = dcs.LanguageIsGL(language) + } + + var books []interface{} + if val, ok := (*dm.Metadata)["books"].([]interface{}); ok { + books = val + } + + var alignmentCounts map[string]interface{} + if val, ok := (*dm.Metadata)["alignment_counts"].(map[string]interface{}); ok { + alignmentCounts = val + } + + var ingredients []interface{} + if val, ok := (*dm.Metadata)["projects"].([]interface{}); ok { + ingredients = val + } + return &api.CatalogV5{ ID: dm.ID, Self: dm.APIURLV5(), @@ -126,12 +168,14 @@ func ToCatalogV5(dm *models.Door43Metadata, mode perm.AccessMode) *api.CatalogV5 ZipballURL: dm.GetZipballURL(), GitTreesURL: dm.GetGitTreesURL(), ContentsURL: dm.GetContentsURL(), - Language: (*dm.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["identifier"].(string), - LanguageTitle: (*dm.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["title"].(string), - LanguageDir: (*dm.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["direction"].(string), + Language: language, + LanguageTitle: languageTitle, + LanguageDir: languageDir, + LanguageIsGL: languageIsGL, Subject: (*dm.Metadata)["dublin_core"].(map[string]interface{})["subject"].(string), Title: (*dm.Metadata)["dublin_core"].(map[string]interface{})["title"].(string), - Books: dm.GetBooks(), + Books: books, + AlignmentCounts: alignmentCounts, BranchOrTag: dm.BranchOrTag, Stage: dm.Stage.String(), Released: dm.ReleaseDateUnix.AsTime(), @@ -139,6 +183,6 @@ func ToCatalogV5(dm *models.Door43Metadata, mode perm.AccessMode) *api.CatalogV5 MetadataURL: dm.GetMetadataURL(), MetadataJSONURL: dm.GetMetadataJSONURL(), MetadataAPIContentsURL: dm.GetMetadataAPIContentsURL(), - Ingredients: (*dm.Metadata)["projects"].([]interface{}), + Ingredients: ingredients, } } diff --git a/modules/convert/release.go b/modules/convert/release.go index 955d3ff05f..0c0e0f6c7c 100644 --- a/modules/convert/release.go +++ b/modules/convert/release.go @@ -16,6 +16,7 @@ func ToRelease(r *models.Release) *api.Release { for _, att := range r.Attachments { assets = append(assets, ToReleaseAttachment(att)) } + return &api.Release{ ID: r.ID, TagName: r.TagName, diff --git a/modules/convert/repository.go b/modules/convert/repository.go index 55b2d00746..6fa1875154 100644 --- a/modules/convert/repository.go +++ b/modules/convert/repository.go @@ -117,18 +117,52 @@ func innerToRepo(repo *repo_model.Repository, mode perm.AccessMode, isParent boo // } var language, languageTitle, languageDir, title, subject, checkingLevel string + var languageIsGL bool var books []string + var alignmentCounts map[string]interface{} if metadata != nil { - language = (*metadata.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["identifier"].(string) - languageTitle = (*metadata.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["title"].(string) - languageDir = (*metadata.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["direction"].(string) title = (*metadata.Metadata)["dublin_core"].(map[string]interface{})["title"].(string) subject = (*metadata.Metadata)["dublin_core"].(map[string]interface{})["subject"].(string) - books = metadata.GetBooks() + + if val, ok := (*metadata.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["identifier"].(string); ok { + language = val + } + + if val, ok := (*metadata.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["direction"].(string); ok { + languageDir = val + } else if language != "" { + languageDir = dcs.GetLanguageDirection(language) + } + + if val, ok := (*metadata.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["title"].(string); ok { + languageTitle = val + } else if language != "" { + languageTitle = dcs.GetLanguageTitle(language) + } + + if val, ok := (*metadata.Metadata)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["is_gl"].(bool); ok { + languageIsGL = val + } else { + languageIsGL = dcs.LanguageIsGL(language) + } + + if val, ok := (*metadata.Metadata)["books"].([]string); ok { + books = val + } else { + books = metadata.GetBooks() + } + + if val, ok := (*metadata.Metadata)["alignment_counts"]; ok { + alignmentCounts = val.(map[string]interface{}) + } + checkingLevel = fmt.Sprintf("%v", (*metadata.Metadata)["checking"].(map[string]interface{})["checking_level"]) } else { - language = dcs.GetLanguageFromRepoName(repo.LowerName) subject = dcs.GetSubjectFromRepoName(repo.LowerName) + language = dcs.GetLanguageFromRepoName(repo.LowerName) + languageTitle = dcs.GetLanguageTitle(language) + languageDir = dcs.GetLanguageDirection(language) + languageIsGL = dcs.LanguageIsGL(language) } /*** END DCS Customizations ***/ @@ -203,9 +237,11 @@ func innerToRepo(repo *repo_model.Repository, mode perm.AccessMode, isParent boo Language: language, LanguageTitle: languageTitle, LanguageDir: languageDir, + LanguageIsGL: languageIsGL, Title: title, Subject: subject, Books: books, + AlignmentCounts: alignmentCounts, CheckingLevel: checkingLevel, Internal: !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePrivate, MirrorInterval: mirrorInterval, diff --git a/modules/dcs/languages.go b/modules/dcs/languages.go index 0491cf6b36..fe36648d05 100644 --- a/modules/dcs/languages.go +++ b/modules/dcs/languages.go @@ -50,3 +50,36 @@ func IsValidLanguage(lang string) bool { _, ok := ln[lang] return ok } + +// GetLanguageDirection returns the language direction +func GetLanguageDirection(lang string) string { + ln := GetLangNames() + if data, ok := ln[lang]; ok { + if val, ok := data.(map[string]interface{})["ld"].(string); ok { + return val + } + } + return "ltr" +} + +// GetLanguageTitle returns the language title +func GetLanguageTitle(lang string) string { + ln := GetLangNames() + if data, ok := ln[lang]; ok { + if val, ok := data.(map[string]interface{})["ln"].(string); ok { + return val + } + } + return "" +} + +// LanguageIsGL returns true if string is a valid language and is a GL +func LanguageIsGL(lang string) bool { + ln := GetLangNames() + if data, ok := ln[lang]; ok { + if val, ok := data.(map[string]interface{})["gw"].(bool); ok { + return val + } + } + return false +} diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index 1dba9aaf37..7e87328b36 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -35,7 +35,7 @@ func (Renderer) NeedPostProcess() bool { return false } // Extensions implements markup.Renderer func (Renderer) Extensions() []string { - return []string{".csv"} // DCS Customization - removes .tsv and adds custom tsv parser + return []string{".csv"} // DCS Customizations - removes .tsv and adds custom tsv parser } // SanitizerRules implements markup.Renderer diff --git a/modules/structs/catalog_next.go b/modules/structs/catalog_next.go index a2dbef3bd2..49c1f3124a 100644 --- a/modules/structs/catalog_next.go +++ b/modules/structs/catalog_next.go @@ -89,31 +89,33 @@ type CatalogV4 struct { // CatalogV5 represents a repository's metadata of a tag or default branch for V5 type CatalogV5 struct { - ID int64 `json:"id"` - Self string `json:"url"` - Name string `json:"name"` - Owner string `json:"owner"` - FullName string `json:"full_name"` - Repo *Repository `json:"repo"` - Release *Release `json:"release"` - TarballURL string `json:"tarbar_url"` - ZipballURL string `json:"zipball_url"` - GitTreesURL string `json:"git_trees_url"` - ContentsURL string `json:"contents_url"` - Language string `json:"language"` - LanguageTitle string `json:"language_title"` - LanguageDir string `json:"language_direction"` - Subject string `json:"subject"` - Title string `json:"title"` - BranchOrTag string `json:"branch_or_tag_name"` - Stage string `json:"stage"` - MetadataURL string `json:"metadata_url"` - MetadataJSONURL string `json:"metadata_json_url"` - MetadataAPIContentsURL string `json:"metadata_api_contents_url"` - MetadataVersion string `json:"metadata_version"` - Released time.Time `json:"released"` - Books []string `json:"books"` - Ingredients []interface{} `json:"ingredients,omitempty"` + ID int64 `json:"id"` + Self string `json:"url"` + Name string `json:"name"` + Owner string `json:"owner"` + FullName string `json:"full_name"` + Repo *Repository `json:"repo"` + Release *Release `json:"release"` + TarballURL string `json:"tarbar_url"` + ZipballURL string `json:"zipball_url"` + GitTreesURL string `json:"git_trees_url"` + ContentsURL string `json:"contents_url"` + Language string `json:"language"` + LanguageTitle string `json:"language_title"` + LanguageDir string `json:"language_direction"` + LanguageIsGL bool `json:"language_is_gl"` + Subject string `json:"subject"` + Title string `json:"title"` + BranchOrTag string `json:"branch_or_tag_name"` + Stage string `json:"stage"` + MetadataURL string `json:"metadata_url"` + MetadataJSONURL string `json:"metadata_json_url"` + MetadataAPIContentsURL string `json:"metadata_api_contents_url"` + MetadataVersion string `json:"metadata_version"` + Released time.Time `json:"released"` + Books []interface{} `json:"books,omitempty"` + AlignmentCounts map[string]interface{} `json:"alignment_counts,omitempty"` + Ingredients []interface{} `json:"ingredients,omitempty"` } // CatalogSearchResultsV3 results of a successful search for V3 diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 0a874a3dc8..d28d047b0d 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -92,18 +92,22 @@ type Repository struct { DefaultMergeStyle string `json:"default_merge_style"` AvatarURL string `json:"avatar_url"` Internal bool `json:"internal"` - Language string `json:"language"` - LanguageTitle string `json:"language_title"` - LanguageDir string `json:"language_direction"` - Subject string `json:"subject"` - Books []string `json:"books"` - Title string `json:"title"` - CheckingLevel string `json:"checking_level"` - Catalog *CatalogStages `json:"catalog"` MirrorInterval string `json:"mirror_interval"` // swagger:strfmt date-time MirrorUpdated time.Time `json:"mirror_updated,omitempty"` RepoTransfer *RepoTransfer `json:"repo_transfer"` + /*** DCS Customizations ***/ + Language string `json:"language"` + LanguageTitle string `json:"language_title"` + LanguageDir string `json:"language_direction"` + LanguageIsGL bool `json:"language_is_gl"` + Subject string `json:"subject"` + Books []string `json:"books,omitempty"` + AlignmentCounts map[string]interface{} `json:"alignment_counts,omitempty"` + Title string `json:"title"` + CheckingLevel string `json:"checking_level"` + Catalog *CatalogStages `json:"catalog"` + /*** END DCS Customizations ***/ } // CreateRepoOption options when creating repository diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 38da03aa9d..fd3e5517d2 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2238,8 +2238,8 @@ topic.done = Done topic.count_prompt = You can not select more than 25 topics topic.format_prompt = Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long. -;;; DCS Customizations [repo.metadatas] -metadatas.desc = Track tagged version and master branch metadata. +;;; DCS Customizations [repo.metadata] +metadata.desc = Track tagged version and master branch metadata. metadata.metadatas = Metadatas metadata.new_metadata = New Metadata metadata.edit = edit diff --git a/routers/api/catalog/api.go b/routers/api/catalog/api.go index 583b6fc449..800f3b88c9 100644 --- a/routers/api/catalog/api.go +++ b/routers/api/catalog/api.go @@ -87,7 +87,7 @@ var versions = []string{ "v4", "v5", } -var latestVersion = versions[len(versions)-1] +var LatestVersion = versions[len(versions)-1] // AllRoutes call all the other route functions for the catalog api func AllRoutes(r *web.Route) { @@ -160,10 +160,18 @@ func LatestRoutes() *web.Route { m.Group("", func() { m.Get("", func(ctx *context.APIContext) { - ctx.Redirect(fmt.Sprintf("/api/catalog/%s", latestVersion)) + var query string + if ctx.Req.URL.RawQuery != "" { + query = "?" + ctx.Req.URL.RawQuery + } + ctx.Redirect(fmt.Sprintf("/api/catalog/%s%s", LatestVersion, query)) }) m.Get("/*", func(ctx *context.APIContext) { - ctx.Redirect(fmt.Sprintf("/api/catalog/%s/%s", latestVersion, ctx.Params("*"))) + var query string + if ctx.Req.URL.RawQuery != "" { + query = "?" + ctx.Req.URL.RawQuery + } + ctx.Redirect(fmt.Sprintf("/api/catalog/%s/%s%s", LatestVersion, ctx.Params("*"), query)) }) }, sudo()) diff --git a/routers/api/catalog/catalog.go b/routers/api/catalog/catalog.go index ac565ea5ac..4b85d742ca 100644 --- a/routers/api/catalog/catalog.go +++ b/routers/api/catalog/catalog.go @@ -27,7 +27,7 @@ func ListCatalogVersionEndpoints(ctx *context.APIContext) { // "$ref": "#/responses/validationError" versionEndpoints := structs.CatalogVersionEndpoints{ - Latest: latestVersion, + Latest: LatestVersion, Versions: map[string]string{}, } diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 8b00b6ef34..d298fbfcde 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -212,10 +212,10 @@ func reqToken() func(ctx *context.APIContext) { func reqExploreSignIn() func(ctx *context.APIContext) { return func(ctx *context.APIContext) { if setting.Service.Explore.RequireSigninView && !ctx.IsSigned { - /*** DCS Customization ***/ + /*** DCS Customizations ***/ return //ctx.Error(http.StatusUnauthorized, "reqExploreSignIn", "you must be signed in to search for users") - /*** END DCS Customization ***/ + /*** END DCS Customizations ***/ } } } diff --git a/services/door43metadata/door43metadata.go b/services/door43metadata/door43metadata.go index 3eb2d764e4..0d945ebc2c 100644 --- a/services/door43metadata/door43metadata.go +++ b/services/door43metadata/door43metadata.go @@ -5,17 +5,32 @@ package door43metadata import ( + "bytes" + "encoding/json" "fmt" + "io" + "io/ioutil" + "net/http" "reflect" + "regexp" + "strings" + "time" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/charset" + "code.gitea.io/gitea/modules/dcs" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" + "github.com/google/uuid" "github.com/mitchellh/mapstructure" "github.com/unknwon/com" "xorm.io/xorm" @@ -43,13 +58,13 @@ func GenerateDoor43Metadata(x *xorm.Engine) error { return err } - cacheRepos := make(map[int64]*repo.Repository) + cacheRepos := make(map[int64]*repo_model.Repository) for _, record := range records { releaseID := com.StrTo(record["release_id"]).MustInt64() repoID := com.StrTo(record["repo_id"]).MustInt64() if cacheRepos[repoID] == nil { - cacheRepos[repoID], err = repo.GetRepositoryByID(repoID) + cacheRepos[repoID], err = repo_model.GetRepositoryByID(repoID) if err != nil { log.Warn("GetRepositoryByID Error: %v\n", err) continue @@ -126,7 +141,7 @@ func ConvertGenericMapToRC020Manifest(manifest *map[string]interface{}) (*struct } // ProcessDoor43MetadataForRepo handles the metadata for a given repo for all its releases -func ProcessDoor43MetadataForRepo(repo *repo.Repository) error { +func ProcessDoor43MetadataForRepo(repo *repo_model.Repository) error { if repo == nil { return fmt.Errorf("no repository provided") } @@ -151,14 +166,14 @@ func ProcessDoor43MetadataForRepo(repo *repo.Repository) error { if releaseID > 0 { release, err = models.GetReleaseByID(releaseID) if err != nil { - fmt.Printf("GetReleaseByID Error: %v\n", err) + log.Error("GetReleaseByID Error: %v\n", err) continue } releaseRef = release.TagName } log.Info("Processing Metadata for repo %s (%d), %s (%d)\n", repo.Name, repo.ID, releaseRef, releaseID) if err = ProcessDoor43MetadataForRepoRelease(repo, release); err != nil { - log.Warn("Error processing metadata for repo %s (%d), %s (%d): %v\n", repo.Name, repo.ID, releaseRef, releaseID, err) + log.Error("Error processing metadata for repo %s (%d), %s (%d): %v\n", repo.Name, repo.ID, releaseRef, releaseID, err) } else { log.Info("Processed Metadata for repo %s (%d), %s (%d)\n", repo.Name, repo.ID, releaseRef, releaseID) } @@ -166,8 +181,62 @@ func ProcessDoor43MetadataForRepo(repo *repo.Repository) error { return nil } +func GetBookAlignmentCount(bookPath string, commit *git.Commit) (int, error) { + blob, err := commit.GetBlobByPath(bookPath) + if err != nil { + log.Warn("GetBlobByPath(%s) Error: %v\n", bookPath, err) + return 0, err + } + dataRc, err := blob.DataAsync() + if err != nil { + log.Error("blob.DataAsync() Error: %v\n", err) + return 0, err + } + defer dataRc.Close() + + buf := make([]byte, 1024) + n, _ := util.ReadAtMost(dataRc, buf) + buf = buf[:n] + + rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc)) + buf, err = io.ReadAll(rd) + if err != nil { + log.Error("io.ReadAll Error: %v", err) + return 0, err + } + matches := regexp.MustCompile(`\\zaln-s`).FindAllStringIndex(string(buf), -1) + return len(matches), nil +} + +// GetAlignmentsCounts get all the alignment counts for all books +func GetAlignmentsCounts(manifest *map[string]interface{}, commit *git.Commit) map[string]int { + var counts = map[string]int{} + if (*manifest)["dublin_core"].(map[string]interface{})["subject"].(string) != "Aligned Bible" || len((*manifest)["projects"].([]interface{})) <= 0 { + return counts + } + for _, prod := range (*manifest)["projects"].([]interface{}) { + bookPath := prod.(map[string]interface{})["path"].(string) + if strings.HasSuffix(bookPath, ".usfm") { + count, _ := GetBookAlignmentCount(bookPath, commit) + counts[prod.(map[string]interface{})["identifier"].(string)] = count + } + } + return counts +} + +// GetBooks get the books of the manifest +func GetBooks(manifest *map[string]interface{}) []string { + var books []string + if len((*manifest)["projects"].([]interface{})) > 0 { + for _, prod := range (*manifest)["projects"].([]interface{}) { + books = append(books, prod.(map[string]interface{})["identifier"].(string)) + } + } + return books +} + // ProcessDoor43MetadataForRepoRelease handles the metadata for a given repo by release based on if the container is a valid RC or not -func ProcessDoor43MetadataForRepoRelease(repo *repo.Repository, release *models.Release) error { +func ProcessDoor43MetadataForRepoRelease(repo *repo_model.Repository, release *models.Release) error { if repo == nil { return fmt.Errorf("no repository provided") } @@ -177,7 +246,7 @@ func ProcessDoor43MetadataForRepoRelease(repo *repo.Repository, release *models. gitRepo, err := git.OpenRepository(repo.RepoPath()) if err != nil { - fmt.Printf("OpenRepository Error: %v\n", err) + log.Error("OpenRepository Error: %v\n", err) return err } defer gitRepo.Close() @@ -203,6 +272,10 @@ func ProcessDoor43MetadataForRepoRelease(repo *repo.Repository, release *models. } } + if release != nil { + UnpackJSONAttachments(release) + } + blob, err := commit.GetBlobByPath("manifest.yaml") if err != nil && !git.IsErrNotExist(err) { return err @@ -278,6 +351,12 @@ func ProcessDoor43MetadataForRepoRelease(repo *repo.Repository, release *models. return models.DeleteDoor43Metadata(dm) } } else { + if (*manifest)["dublin_core"].(map[string]interface{})["subject"].(string) == "Aligned Bible" { + (*manifest)["alignment_counts"] = GetAlignmentsCounts(manifest, commit) + } + (*manifest)["books"] = GetBooks(manifest) + lc := (*manifest)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["identifier"].(string) + (*manifest)["dublin_core"].(map[string]interface{})["language"].(map[string]interface{})["is_gl"] = dcs.LanguageIsGL(lc) log.Warn("%s/%s: manifest.yaml is valid.", repo.FullName(), branchOrTag) if dm == nil { dm = &models.Door43Metadata{ @@ -297,9 +376,110 @@ func ProcessDoor43MetadataForRepoRelease(repo *repo.Repository, release *models. dm.ReleaseDateUnix = releaseDateUnix dm.Stage = stage dm.BranchOrTag = branchOrTag - return models.UpdateDoor43MetadataCols(dm, "metadata", "release_date_unix", "stage", "branch_or_tag") + return models.UpdateDoor43MetadataCols(dm, "lang", "metadata", "release_date_unix", "stage", "branch_or_tag") } } return nil } + +func UnpackJSONAttachments(release *models.Release) { + if release == nil || len(release.Attachments) == 0 { + return + } + var jsonFileNameSuffix = regexp.MustCompile(`(file|link)s*\.json$`) + for _, attachment := range release.Attachments { + if jsonFileNameSuffix.MatchString(attachment.Name) { + remoteAttachments, err := GetAttachmentsFromJSON(attachment) + if err != nil { + log.Warn("GetAttachmentsFromJSON Error: %v", err) + continue + } + for _, remoteAttachment := range remoteAttachments { + remoteAttachment.ReleaseID = attachment.ReleaseID + remoteAttachment.RepoID = attachment.RepoID + remoteAttachment.UploaderID = attachment.UploaderID + var foundExisting = false + for _, a := range release.Attachments { + if a.Name == remoteAttachment.Name { + if remoteAttachment.Size > 0 { + a.Size = remoteAttachment.Size + } + if remoteAttachment.BrowserDownloadURL != "" { + a.BrowserDownloadURL = remoteAttachment.BrowserDownloadURL + } + a.BrowserDownloadURL = remoteAttachment.BrowserDownloadURL + if err := repo_model.UpdateAttachment(a); err != nil { + log.Warn("UpdateAttachment [%d]: %v", a.ID, err) + continue + } + foundExisting = true + break + } + } + if foundExisting { + continue + } + // No existing attachment was found with the same name, so we insert a new one + remoteAttachment.UUID = uuid.New().String() + if _, err = db.GetEngine(db.DefaultContext).Insert(remoteAttachment); err != nil { + log.Warn("insert attachment [%d]: %v", remoteAttachment.ID, err) + continue + } + } + if err := repo_model.DeleteAttachment(attachment, true); err != nil { + log.Error("delete attachment [%d]: %v", attachment.ID, err) + continue + } + continue + } + } +} + +// GetAttachmentsFromJSON gets the attachments from uploaded +func GetAttachmentsFromJSON(attachment *repo_model.Attachment) ([]*repo_model.Attachment, error) { + var url string + if setting.Attachment.ServeDirect { + //If we have a signed url (S3, object storage), redirect to this directly. + urlObj, err := storage.Attachments.URL(attachment.RelativePath(), attachment.Name) + + if urlObj != nil && err == nil { + url = urlObj.String() + } + } else { + url = attachment.DownloadURL() + } + client := http.Client{ + Timeout: time.Second * 2, // Timeout after 2 seconds + } + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("http.NewRequest Error: %v", err) + } + req.Header.Set("User-Agent", "dcs") + res, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("client.Do Error: %v", err) + } + if res.StatusCode != 200 { + return nil, fmt.Errorf("client.Do Error: `%s` returned StatusCode [%d]", attachment.DownloadURL(), res.StatusCode) + } + if res.Body != nil { + defer res.Body.Close() + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("ioutil.ReadAll Error: %v", err) + } + attachments := []*repo_model.Attachment{} + if err1 := json.Unmarshal(body, &attachments); err1 != nil { + // We couldn't unmarshal an array of attachments, so lets see if it is just a single attachment + attachment := &repo_model.Attachment{} + if err2 := json.Unmarshal(body, attachment); err2 != nil { + return nil, fmt.Errorf("json.Unmarshal Error: %v", err1) + } + attachments = append(attachments, attachment) + } + return attachments, nil +} diff --git a/templates/repo/home.tmpl b/templates/repo/home.tmpl index 28353255b3..ff7cba7f58 100644 --- a/templates/repo/home.tmpl +++ b/templates/repo/home.tmpl @@ -126,6 +126,20 @@ {{svg "gitea-vscode" 16 "mr-3"}}{{.i18n.Tr "repo.clone_in_vsc"}} + {{end}} diff --git a/templates/repo/release/new.tmpl b/templates/repo/release/new.tmpl index 9193389e64..ef5fcfb28d 100644 --- a/templates/repo/release/new.tmpl +++ b/templates/repo/release/new.tmpl @@ -65,7 +65,9 @@
- + + + {{.Size | FileSize}} diff --git a/templates/repo/sub_menu.tmpl b/templates/repo/sub_menu.tmpl index a9cb532a2e..65011a6286 100644 --- a/templates/repo/sub_menu.tmpl +++ b/templates/repo/sub_menu.tmpl @@ -21,7 +21,7 @@ {{$dm := (GetDefaultBranchMetadata .Repository.ID)}} {{ if $dm }}
- {{$dm.Metadata.dublin_core.language.title}} ({{$dm.Metadata.dublin_core.language.identifier}}) + {{$dm.Metadata.dublin_core.language.title}} ({{$dm.Metadata.dublin_core.language.identifier}}){{if $dm.Metadata.dublin_core.language.is_gl}}*{{end}}
{{ end }} diff --git a/templates/swagger/catalog/catalog_json.tmpl b/templates/swagger/catalog/catalog_json.tmpl index 215df17a09..6ebe0c50ae 100644 --- a/templates/swagger/catalog/catalog_json.tmpl +++ b/templates/swagger/catalog/catalog_json.tmpl @@ -1659,10 +1659,17 @@ "description": "CatalogV5 represents a repository's metadata of a tag or default branch for V5", "type": "object", "properties": { + "alignment_counts": { + "type": "object", + "additionalProperties": { + "type": "object" + }, + "x-go-name": "AlignmentCounts" + }, "books": { "type": "array", "items": { - "type": "string" + "type": "object" }, "x-go-name": "Books" }, @@ -1702,6 +1709,10 @@ "type": "string", "x-go-name": "LanguageDir" }, + "language_is_gl": { + "type": "boolean", + "x-go-name": "LanguageIsGL" + }, "language_title": { "type": "string", "x-go-name": "LanguageTitle" @@ -2070,6 +2081,13 @@ "description": "Repository represents a repository", "type": "object", "properties": { + "alignment_counts": { + "type": "object", + "additionalProperties": { + "type": "object" + }, + "x-go-name": "AlignmentCounts" + }, "allow_merge_commits": { "type": "boolean", "x-go-name": "AllowMerge" @@ -2189,6 +2207,7 @@ "$ref": "#/definitions/InternalTracker" }, "language": { + "description": "DCS Customizations ***/", "type": "string", "x-go-name": "Language" }, @@ -2196,6 +2215,10 @@ "type": "string", "x-go-name": "LanguageDir" }, + "language_is_gl": { + "type": "boolean", + "x-go-name": "LanguageIsGL" + }, "language_title": { "type": "string", "x-go-name": "LanguageTitle" diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 161c368890..d2e2661d4b 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -17234,6 +17234,13 @@ "description": "Repository represents a repository", "type": "object", "properties": { + "alignment_counts": { + "type": "object", + "additionalProperties": { + "type": "object" + }, + "x-go-name": "AlignmentCounts" + }, "allow_merge_commits": { "type": "boolean", "x-go-name": "AllowMerge" @@ -17353,6 +17360,7 @@ "$ref": "#/definitions/InternalTracker" }, "language": { + "description": "DCS Customizations ***/", "type": "string", "x-go-name": "Language" }, @@ -17360,6 +17368,10 @@ "type": "string", "x-go-name": "LanguageDir" }, + "language_is_gl": { + "type": "boolean", + "x-go-name": "LanguageIsGL" + }, "language_title": { "type": "string", "x-go-name": "LanguageTitle"