revise/simplify 'is-not-exist'; ldp.reader to honor 'sync'

* 'is-not-exist': remove redundant variations, remove redundant checks
  - unify usage and move to 'cos' package (where it belongs)
* 'err-not-found': revise as ('where' and 'what') was not found
* 'ldp.reader': when remote metadata check returns 'not-found':
  - delete in-cluster copy when 'sync' requested

Signed-off-by: Alex Aizman <[email protected]>

ais/backend/ais.go

@@ -365,7 +365,7 @@ func (m *AISBackendProvider) resolve(uuid string) (*remAis, string, error) {
 	}
 	alias := uuid
 	if uuid, ok = m.alias[alias]; !ok {
-		return nil, "", cos.NewErrNotFound("%s: remote cluster %q", m.t, alias)
+		return nil, "", cos.NewErrNotFound(m.t, "remote cluster \""+alias+"\"")
 	}
 	remAis, ok = m.remote[uuid]
 	debug.Assertf(ok, "%q vs %q", alias, uuid)

ais/backend/gcp.go

@@ -466,13 +466,12 @@ func _gcpErr(gcpError error) error {
 
 func handleObjectError(ctx context.Context, gcpClient *storage.Client, objErr error, bck *cmn.Bck) (int, error) {
 	if objErr != storage.ErrObjectNotExist {
-		return http.StatusBadRequest, objErr
+		return http.StatusBadRequest, _gcpErr(objErr)
 	}
 
-	// Object does not exist, but in GCP it doesn't mean that the bucket existed.
-	// Check if the buckets exists.
+	// Object does not exist but in GCP it doesn't necessarily mean that the bucket does.
 	if _, err := gcpClient.Bucket(bck.Name).Attrs(ctx); err != nil {
 		return gcpErrorToAISError(err, bck)
 	}
-	return http.StatusNotFound, cos.NewErrNotFound(objErr.Error())
+	return http.StatusNotFound, cos.NewErrNotFound(nil, _gcpErr(objErr).Error())
 }

ais/htrun.go

@@ -1232,7 +1232,7 @@ func (h *htrun) writeErrStatusf(w http.ResponseWriter, r *http.Request, errCode
 
 func (h *htrun) writeErrf(w http.ResponseWriter, r *http.Request, format string, a ...any) {
 	err := fmt.Errorf(format, a...)
-	if cmn.IsNotExist(err) {
+	if cos.IsNotExist(err) {
 		h.writeErrMsg(w, r, err.Error(), http.StatusNotFound)
 	} else {
 		h.writeErrMsg(w, r, err.Error())

ais/prxclu.go

@@ -1,6 +1,6 @@
 // Package ais provides core functionality for the AIStore object storage.
 /*
- * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
  */
 package ais
 
@@ -1347,7 +1347,7 @@ func (p *proxy) rmNode(w http.ResponseWriter, r *http.Request, msg *apc.ActMsg)
 	}
 	si := smap.GetNode(opts.DaemonID)
 	if si == nil {
-		err := cos.NewErrNotFound("%s: node %s", p.si, opts.DaemonID)
+		err := cos.NewErrNotFound(p, "node "+opts.DaemonID)
 		p.writeErr(w, r, err, http.StatusNotFound)
 		return
 	}
@@ -1529,7 +1529,7 @@ func (p *proxy) stopMaintenance(w http.ResponseWriter, r *http.Request, msg *apc
 	}
 	si := smap.GetNode(opts.DaemonID)
 	if si == nil {
-		err := cos.NewErrNotFound("%s: node %s", p.si, opts.DaemonID)
+		err := cos.NewErrNotFound(p, "node "+opts.DaemonID)
 		p.writeErr(w, r, err, http.StatusNotFound)
 		return
 	}

ais/prxetl.go

@@ -1,6 +1,6 @@
 // Package ais provides core functionality for the AIStore object storage.
 /*
- * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
  */
 package ais
 
@@ -149,7 +149,7 @@ func (p *proxy) handleETLPost(w http.ResponseWriter, r *http.Request) {
 	etlMD := p.owner.etl.get()
 	etlMsg := etlMD.get(etlName)
 	if etlMsg == nil {
-		p.writeErr(w, r, cos.NewErrNotFound("%s: etl[%s]", p, etlName))
+		p.writeErr(w, r, cos.NewErrNotFound(p, "etl job "+etlName))
 		return
 	}
 
@@ -193,7 +193,7 @@ func (p *proxy) handleETLDelete(w http.ResponseWriter, r *http.Request) {
 func (p *proxy) _deleteETLPre(ctx *etlMDModifier, clone *etlMD) (err error) {
 	debug.AssertNoErr(k8s.ValidateEtlName(ctx.etlName))
 	if exists := clone.del(ctx.etlName); !exists {
-		err = cos.NewErrNotFound("%s: etl[%s]", p, ctx.etlName)
+		err = cos.NewErrNotFound(p, "etl job "+ctx.etlName)
 	}
 	return
 }
@@ -275,7 +275,7 @@ func (p *proxy) infoETL(w http.ResponseWriter, r *http.Request, etlName string)
 	etlMD := p.owner.etl.get()
 	initMsg := etlMD.get(etlName)
 	if initMsg == nil {
-		p.writeErr(w, r, cos.NewErrNotFound("%s: etl[%s]", p, etlName))
+		p.writeErr(w, r, cos.NewErrNotFound(p, "etl job "+etlName))
 		return
 	}
 	p.writeJSON(w, r, initMsg, "info-etl")

ais/prxnotif.go

@@ -185,7 +185,10 @@ func (n *notifs) _finished(nl nl.Listener, tsi *meta.Snode, msg *core.NotifMsg)
 		stats, _, abortedSnap, err := nl.UnmarshalStats(msg.Data)
 		debug.AssertNoErr(err)
 		nl.SetStats(tsi.ID(), stats)
-		debug.Assertf(abortedSnap == msg.AbortedX, "%s: %t vs %t [%s]", msg, abortedSnap, msg.AbortedX, nl.String())
+
+		if abortedSnap != msg.AbortedX && cmn.FastV(4, cos.SmoduleAIS) {
+			nlog.Infof("Warning: %s: %t vs %t [%s]", msg, abortedSnap, msg.AbortedX, nl.String())
+		}
 		aborted = aborted || abortedSnap
 	}
 	if msg.ErrMsg != "" {

ais/preverse.go → ais/prxrev.go

@@ -1,6 +1,6 @@
 // Package ais provides core functionality for the AIStore object storage.
 /*
- * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
  */
 package ais
 
@@ -183,7 +183,7 @@ func (p *proxy) reverseRemAis(w http.ResponseWriter, r *http.Request, msg *apc.A
 		}
 	}
 	if !exists {
-		err = cos.NewErrNotFound("%s: remote UUID/alias %q (%s)", p.si, aliasOrUUID, backend)
+		err = cos.NewErrNotFound(p, "remote UUID/alias "+aliasOrUUID)
 		p.writeErr(w, r, err)
 		return err
 	}

ais/target.go

@@ -972,7 +972,7 @@ func (t *target) objhead(hdr http.Header, query url.Values, bck *meta.Bck, lom *
 
 	if !exists {
 		if bck.IsAIS() || apc.IsFltPresent(fltPresence) {
-			err = cos.NewErrNotFound("%s: object %s", t, lom.Cname())
+			err = cos.NewErrNotFound(t, lom.Cname())
 			return http.StatusNotFound, err
 		}
 	}
@@ -1088,7 +1088,7 @@ func (t *target) httpobjpatch(w http.ResponseWriter, r *http.Request, apireq *ap
 		return
 	}
 	if err := lom.Load(true /*cache it*/, false /*locked*/); err != nil {
-		if cmn.IsObjNotExist(err) {
+		if cos.IsNotExist(err) {
 			t.writeErr(w, r, err, http.StatusNotFound)
 		} else {
 			t.writeErr(w, r, err)
@@ -1269,7 +1269,7 @@ func (t *target) delobj(lom *core.LOM, evict bool) (int, error, bool) {
 	delFromBackend = lom.Bck().IsRemote() && !evict
 	err := lom.Load(false /*cache it*/, true /*locked*/)
 	if err != nil {
-		if !cmn.IsObjNotExist(err) {
+		if !cos.IsNotExist(err) {
 			return 0, err, false
 		}
 		if !delFromBackend {

ais/tgtbck.go

@@ -1,6 +1,6 @@
 // Package ais provides core functionality for the AIStore object storage.
 /*
- * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
  */
 package ais
 
@@ -333,7 +333,7 @@ func (t *target) bsumm(w http.ResponseWriter, r *http.Request, phase string, bck
 
 	// never started
 	if xctn == nil {
-		err := cos.NewErrNotFound("%s: x-%s[%s] (failed to start?)", t, apc.ActSummaryBck, msg.UUID)
+		err := cos.NewErrNotFound(t, apc.ActSummaryBck+" job "+msg.UUID)
 		t._erris(w, r, dpq.silent, err, http.StatusNotFound)
 		return
 	}

ais/tgtobj.go

@@ -512,7 +512,7 @@ func (goi *getOI) get() (errCode int, err error) {
 do:
 	err = goi.lom.Load(true /*cache it*/, true /*locked*/)
 	if err != nil {
-		cold = cmn.IsObjNotExist(err)
+		cold = cos.IsNotExist(err)
 		if !cold {
 			return http.StatusInternalServerError, err
 		}
@@ -602,7 +602,7 @@ do:
 		if res.Err != nil {
 			goi.lom.Unlock(true)
 			goi.unlocked = true
-			if res.ErrCode != http.StatusNotFound && !cmn.IsObjNotExist(res.Err) {
+			if res.ErrCode != http.StatusNotFound && !cos.IsNotExist(res.Err) {
 				nlog.Infoln(ftcg+"(read)", goi.lom.Cname(), res.Err, res.ErrCode)
 			}
 			return res.ErrCode, res.Err
@@ -881,7 +881,7 @@ gfn:
 	if err != nil {
 		err = cmn.NewErrFailedTo(goi.t, "goi-restore-any", goi.lom, err)
 	} else {
-		err = cos.NewErrNotFound("%s: %s", goi.t.si, goi.lom.Cname())
+		err = cos.NewErrNotFound(goi.t, goi.lom.Cname())
 	}
 	errCode = http.StatusNotFound
 	return
@@ -1013,7 +1013,8 @@ func (goi *getOI) fini(fqn string, lmfh *os.File, hdr http.Header, hrng *htrange
 			return
 		}
 		if csl == nil {
-			return http.StatusNotFound, cos.NewErrNotFound("%q in archive %q", goi.archive.filename, goi.lom.Cname())
+			return http.StatusNotFound,
+				cos.NewErrNotFound(goi.t, goi.archive.filename+" in "+goi.lom.Cname())
 		}
 		// found
 		defer func() {
@@ -1402,7 +1403,7 @@ func (coi *copyOI) _regular(t *target, lom, dst *core.LOM) (size int64, _ error)
 	defer lom.Unlock(lcopy)
 
 	if err := lom.Load(false /*cache it*/, true /*locked*/); err != nil {
-		if !cmn.IsObjNotExist(err) {
+		if !cos.IsNotExist(err) {
 			err = cmn.NewErrFailedTo(t, "coi-load", lom, err)
 		}
 		return 0, err

ais/tgts3.go

@@ -1,6 +1,6 @@
 // Package ais provides core functionality for the AIStore object storage.
 /*
- * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
  */
 package ais
 
@@ -151,7 +151,7 @@ func (t *target) copyObjS3(w http.ResponseWriter, r *http.Request, config *cmn.C
 	if err != nil {
 		if err == cmn.ErrSkip {
 			name := lom.Cname()
-			s3.WriteErr(w, r, cos.NewErrNotFound("%s: %s", t.si, name), http.StatusNotFound)
+			s3.WriteErr(w, r, cos.NewErrNotFound(t, name), http.StatusNotFound)
 		} else {
 			s3.WriteErr(w, r, err, 0)
 		}
@@ -297,12 +297,12 @@ func (t *target) headObjS3(w http.ResponseWriter, r *http.Request, items []strin
 	err = lom.Load(true /*cache it*/, false /*locked*/)
 	if err != nil {
 		exists = false
-		if !cmn.IsObjNotExist(err) {
+		if !cos.IsNotExist(err) {
 			s3.WriteErr(w, r, err, 0)
 			return
 		}
 		if bck.IsAIS() {
-			s3.WriteErr(w, r, cos.NewErrNotFound("%s: object %s", t.si, lom.Cname()), 0)
+			s3.WriteErr(w, r, cos.NewErrNotFound(t, lom.Cname()), 0)
 			return
 		}
 	}
@@ -364,7 +364,7 @@ func (t *target) delObjS3(w http.ResponseWriter, r *http.Request, items []string
 	if err != nil {
 		name := lom.Cname()
 		if errCode == http.StatusNotFound {
-			s3.WriteErr(w, r, cos.NewErrNotFound("%s: %s", t.si, name), http.StatusNotFound)
+			s3.WriteErr(w, r, cos.NewErrNotFound(t, name), http.StatusNotFound)
 		} else {
 			s3.WriteErr(w, r, fmt.Errorf("error deleting %s: %v", name, err), errCode)
 		}

ais/txn.go

@@ -1,6 +1,6 @@
 // Package ais provides core functionality for the AIStore object storage.
 /*
- * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
  */
 package ais
 
@@ -188,7 +188,7 @@ func (txns *transactions) find(uuid, act string) (txn, error) {
 	if !ok {
 		// a) not found (benign in an unlikely event of failing to commit)
 		txns.mtx.Unlock()
-		return nil, cos.NewErrNotFound("%s: txn %q", txns.t, uuid)
+		return nil, cos.NewErrNotFound(txns.t, "txn "+uuid)
 	}
 
 	if act == "" {
@@ -256,7 +256,7 @@ func (txns *transactions) commitAfter(caller string, msg *aisMsg, err error, arg
 		rndzvs, ok := txns.rendezvous.m[msg.UUID]
 		if !ok { // can't happen
 			txns.rendezvous.mtx.Unlock()
-			errDone = cos.NewErrNotFound("%s: rendezvous record %q (%v)", txns.t.si, msg.UUID, errDone)
+			errDone = cos.NewErrNotFound(txns.t, "rendezvous record "+msg.UUID)
 			return
 		}
 		rndzvs.err = &txnError{err: err}

cmd/authn/mgr.go

@@ -1,6 +1,6 @@
 // Package authn is authentication server for AIStore.
 /*
- * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
  */
 package main
 
@@ -53,6 +53,8 @@ func newMgr(driver kvdb.Driver) (m *mgr, err error) {
 	return
 }
 
+func (*mgr) String() string { return svcName }
+
 //
 // users ============================================================
 //
@@ -86,7 +88,7 @@ func (m *mgr) updateUser(userID string, updateReq *authn.User) error {
 	uInfo := &authn.User{}
 	err := m.db.Get(usersCollection, userID, uInfo)
 	if err != nil {
-		return cos.NewErrNotFound("%s: user %q", svcName, userID)
+		return cos.NewErrNotFound(m, "user "+userID)
 	}
 	if userID == adminUserID && len(updateReq.Roles) != 0 {
 		return errors.New("cannot change administrator's role")
@@ -176,7 +178,7 @@ func (m *mgr) updateRole(role string, updateReq *authn.Role) error {
 	rInfo := &authn.Role{}
 	err := m.db.Get(rolesCollection, role, rInfo)
 	if err != nil {
-		return cos.NewErrNotFound("%s: role %q", svcName, role)
+		return cos.NewErrNotFound(m, "role "+role)
 	}
 
 	if updateReq.Desc != "" {
@@ -287,7 +289,7 @@ func (m *mgr) cluLookup(cluID, cluAlias string) string {
 func (m *mgr) getCluster(cluID string) (*authn.CluACL, error) {
 	cid := m.cluLookup(cluID, cluID)
 	if cid == "" {
-		return nil, cos.NewErrNotFound("%s: cluster %q", svcName, cluID)
+		return nil, cos.NewErrNotFound(m, "cluster "+cluID)
 	}
 	clu := &authn.CluACL{}
 	err := m.db.Get(clustersCollection, cid, clu)
@@ -350,7 +352,7 @@ func (m *mgr) updateCluster(cluID string, info *authn.CluACL) error {
 func (m *mgr) delCluster(cluID string) error {
 	cid := m.cluLookup(cluID, cluID)
 	if cid == "" {
-		return cos.NewErrNotFound("%s: cluster %q", svcName, cluID)
+		return cos.NewErrNotFound(m, "cluster "+cluID)
 	}
 	return m.db.Delete(clustersCollection, cid)
 }
@@ -386,7 +388,7 @@ func (m *mgr) issueToken(userID, pwd string, msg *authn.LoginMsg) (string, error
 		}
 		cid = m.cluLookup(msg.ClusterID, msg.ClusterID)
 		if cid == "" {
-			return "", cos.NewErrNotFound("%s: cluster %q", svcName, msg.ClusterID)
+			return "", cos.NewErrNotFound(m, "cluster "+msg.ClusterID)
 		}
 		uInfo.ClusterACLs = mergeClusterACLs(make([]*authn.CluACL, 0, len(uInfo.ClusterACLs)), uInfo.ClusterACLs, cid)
 		uInfo.BucketACLs = mergeBckACLs(make([]*authn.BckACL, 0, len(uInfo.BucketACLs)), uInfo.BucketACLs, cid)

cmd/cli/cli/tcbtco.go

@@ -1,6 +1,6 @@
 // Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
 /*
- * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
  */
 package cli
 
@@ -64,9 +64,8 @@ func copyBucketHandler(c *cli.Context) (err error) {
 		bckTo = bckFrom
 	}
 
-	allIncludingRemote := flagIsSet(c, copyAllObjsFlag) || flagIsSet(c, syncFlag)
-
-	return copyTransform(c, "" /*etlName*/, objFrom, bckFrom, bckTo, allIncludingRemote)
+	// NOTE: copyAllObjsFlag forces 'x-list' to list the remote one, and vice versa
+	return copyTransform(c, "" /*etlName*/, objFrom, bckFrom, bckTo, flagIsSet(c, copyAllObjsFlag))
 }
 
 //