Merge pull request #1594 from alicevision/dev/tracksUpgrade

Upgrade tracks to allow more advanced storage and reduce indirections
alicevision · Nov 10, 2023 · 15210d8 · 15210d8
2 parents 23c42b2 + bac8ea8
commit 15210d8
Show file tree

Hide file tree

Showing 13 changed files with 83 additions and 55 deletions.
diff --git a/src/aliceVision/sfm/pipeline/global/GlobalSfMTranslationAveragingSolver.cpp b/src/aliceVision/sfm/pipeline/global/GlobalSfMTranslationAveragingSolver.cpp
@@ -503,7 +503,7 @@ void GlobalSfMTranslationAveragingSolver::ComputePutativeTranslation_EdgesCovera
 
                                         // extract camera indexes
                                         const size_t id_view_I = iter_I->first;
-                                        const size_t id_feat_I = iter_I->second;
+                                        const size_t id_feat_I = iter_I->second.featureId;
 
                                         // loop on subtracks
                                         for (size_t index_J = index_I + 1; index_J < track.featPerView.size(); ++index_J)
@@ -513,7 +513,7 @@ void GlobalSfMTranslationAveragingSolver::ComputePutativeTranslation_EdgesCovera
 
                                             // extract camera indexes
                                             const size_t id_view_J = iter_J->first;
-                                            const size_t id_feat_J = iter_J->second;
+                                            const size_t id_feat_J = iter_J->second.featureId;
 
                                             newpairMatches[std::make_pair(id_view_I, id_view_J)][track.descType].emplace_back(id_feat_I, id_feat_J);
                                         }
@@ -606,7 +606,7 @@ bool GlobalSfMTranslationAveragingSolver::Estimate_T_triplet(const SfMData& sfmD
         for (track::Track::FeatureIdPerView::const_iterator iter = track.featPerView.begin(); iter != track.featPerView.end(); ++iter, ++index)
         {
             const size_t idx_view = iter->first;
-            const feature::PointFeature pt = normalizedFeaturesPerView.getFeatures(idx_view, track.descType)[iter->second];
+            const feature::PointFeature pt = normalizedFeaturesPerView.getFeatures(idx_view, track.descType)[iter->second.featureId];
             xxx[index]->col(cpt) = pt.coords().cast<double>();
             const View* view = sfmData.getViews().at(idx_view).get();
             intrinsic_ids.insert(view->getIntrinsicId());

diff --git a/src/aliceVision/sfm/pipeline/global/ReconstructionEngine_globalSfM.cpp b/src/aliceVision/sfm/pipeline/global/ReconstructionEngine_globalSfM.cpp
@@ -304,7 +304,7 @@ bool ReconstructionEngine_globalSfM::Compute_Initial_Structure(matching::Pairwis
             for (Track::FeatureIdPerView::const_iterator it = track.featPerView.begin(); it != track.featPerView.end(); ++it)
             {
                 const size_t imaIndex = it->first;
-                const size_t featIndex = it->second;
+                const size_t featIndex = it->second.featureId;
                 const PointFeature& pt = _featuresPerView->getFeatures(imaIndex, track.descType)[featIndex];
 
                 const double scale = (_featureConstraint == EFeatureConstraint::BASIC) ? 0.0 : pt.scale();

diff --git a/src/aliceVision/sfm/pipeline/sequential/ReconstructionEngine_sequentialSfM.cpp b/src/aliceVision/sfm/pipeline/sequential/ReconstructionEngine_sequentialSfM.cpp
@@ -108,7 +108,7 @@ void computeTracksPyramidPerView(const track::TracksPerView& tracksPerView,
         {
             const std::size_t trackId = viewTracks.second[i];
             const track::Track& track = map_tracks.at(trackId);
-            const std::size_t featIndex = track.featPerView.at(viewId);
+            const std::size_t featIndex = track.featPerView.at(viewId).featureId;
             const auto& feature = featuresProvider.getFeatures(viewId, track.descType)[featIndex];
 
             for (std::size_t level = 0; level < pyramidDepth; ++level)
@@ -423,7 +423,7 @@ void ReconstructionEngine_sequentialSfM::remapLandmarkIdsToTrackIds()
 
         for (const auto& featView : track.featPerView)
         {
-            const ObsToLandmark::const_iterator it = obsToLandmark.find(ObsKey(featView.first, featView.second, track.descType));
+            const ObsToLandmark::const_iterator it = obsToLandmark.find(ObsKey(featView.first, featView.second.featureId, track.descType));
 
             if (it != obsToLandmark.end())
             {
@@ -1208,8 +1208,8 @@ bool ReconstructionEngine_sequentialSfM::makeInitialPair3D(const Pair& currentPa
     {
         assert(iterT->second.featPerView.size() == 2);
         auto iter = iterT->second.featPerView.begin();
-        const std::size_t i = iter->second;
-        const std::size_t j = (++iter)->second;
+        const std::size_t i = iter->second.featureId;
+        const std::size_t j = (++iter)->second.featureId;
 
         Vec2 feat = _featuresPerView->getFeatures(I, iterT->second.descType)[i].coords().cast<double>();
         xI.col(cptIndex) = camI->get_ud_pixel(feat);
@@ -1397,8 +1397,8 @@ bool ReconstructionEngine_sequentialSfM::getBestInitialImagePairs(std::vector<Pa
         for (aliceVision::track::TracksMap::const_iterator iterT = map_tracksCommon.begin(); iterT != map_tracksCommon.end(); ++iterT, ++cptIndex)
         {
             auto iter = iterT->second.featPerView.begin();
-            const size_t i = iter->second;
-            const size_t j = (++iter)->second;
+            const size_t i = iter->second.featureId;
+            const size_t j = (++iter)->second.featureId;
             commonTracksIds[cptIndex] = iterT->first;
 
             const auto& viewI = _featuresPerView->getFeatures(I, iterT->second.descType);
@@ -1441,8 +1441,8 @@ bool ReconstructionEngine_sequentialSfM::getBestInitialImagePairs(std::vector<Pa
                 multiview::TriangulateDLT(PI, xI.col(inlier_idx), PJ, xJ.col(inlier_idx), X);
                 IndexT trackId = commonTracksIds[inlier_idx];
                 auto iter = map_tracksCommon[trackId].featPerView.begin();
-                const Vec2 featI = _featuresPerView->getFeatures(I, map_tracksCommon[trackId].descType)[iter->second].coords().cast<double>();
-                const Vec2 featJ = _featuresPerView->getFeatures(J, map_tracksCommon[trackId].descType)[(++iter)->second].coords().cast<double>();
+                const Vec2 featI = _featuresPerView->getFeatures(I, map_tracksCommon[trackId].descType)[iter->second.featureId].coords().cast<double>();
+                const Vec2 featJ = _featuresPerView->getFeatures(J, map_tracksCommon[trackId].descType)[(++iter)->second.featureId].coords().cast<double>();
                 vec_angles[i] = angleBetweenRays(pose_I, camI, pose_J, camJ, featI, featJ);
                 validCommonTracksIds[i] = trackId;
                 ++i;
@@ -1771,7 +1771,7 @@ ObservationData getObservationData(const SfMData& scene, feature::FeaturesPerVie
     Pose3 pose = scene.getPose(*view).getTransform();
     Mat34 P = camPinHole->getProjectiveEquivalent(pose);
 
-    const auto& feature = featuresPerView->getFeatures(viewId, track.descType)[track.featPerView.at(viewId)];
+    const auto& feature = featuresPerView->getFeatures(viewId, track.descType)[track.featPerView.at(viewId).featureId];
     Vec2 x = feature.coords().cast<double>();
     Vec2 xUd = cam->get_ud_pixel(x);  // undistorted 2D point
 
@@ -1906,10 +1906,10 @@ void ReconstructionEngine_sequentialSfM::triangulate_multiViewsLORANSAC(SfMData&
             landmark.descType = track.descType;
             for (const IndexT& viewId : inliers)  // add inliers as observations
             {
-                const Vec2 x = _featuresPerView->getFeatures(viewId, track.descType)[track.featPerView.at(viewId)].coords().cast<double>();
-                const feature::PointFeature& p = _featuresPerView->getFeatures(viewId, track.descType)[track.featPerView.at(viewId)];
+                const Vec2 x = _featuresPerView->getFeatures(viewId, track.descType)[track.featPerView.at(viewId).featureId].coords().cast<double>();
+                const feature::PointFeature& p = _featuresPerView->getFeatures(viewId, track.descType)[track.featPerView.at(viewId).featureId];
                 const double scale = (_params.featureConstraint == EFeatureConstraint::BASIC) ? 0.0 : p.scale();
-                landmark.observations[viewId] = Observation(x, track.featPerView.at(viewId), scale);
+                landmark.observations[viewId] = Observation(x, track.featPerView.at(viewId).featureId, scale);
             }
 #pragma omp critical
             {
@@ -1994,11 +1994,11 @@ void ReconstructionEngine_sequentialSfM::triangulate_2Views(SfMData& scene,
                 const std::size_t trackId = trackIt.first;
                 const track::Track& track = trackIt.second;
 
-                const Vec2 xI = _featuresPerView->getFeatures(I, track.descType)[track.featPerView.at(I)].coords().cast<double>();
-                const Vec2 xJ = _featuresPerView->getFeatures(J, track.descType)[track.featPerView.at(J)].coords().cast<double>();
+                const Vec2 xI = _featuresPerView->getFeatures(I, track.descType)[track.featPerView.at(I).featureId].coords().cast<double>();
+                const Vec2 xJ = _featuresPerView->getFeatures(J, track.descType)[track.featPerView.at(J).featureId].coords().cast<double>();
 
-                const feature::PointFeature& featI = _featuresPerView->getFeatures(I, track.descType)[track.featPerView.at(I)];
-                const feature::PointFeature& featJ = _featuresPerView->getFeatures(J, track.descType)[track.featPerView.at(J)];
+                const feature::PointFeature& featI = _featuresPerView->getFeatures(I, track.descType)[track.featPerView.at(I).featureId];
+                const feature::PointFeature& featJ = _featuresPerView->getFeatures(J, track.descType)[track.featPerView.at(J).featureId];
                 // test if the track already exists in 3D
                 bool trackIdExists;
 #pragma omp critical
@@ -2021,7 +2021,7 @@ void ReconstructionEngine_sequentialSfM::triangulate_2Views(SfMData& scene,
                             if (poseI.depth(landmark.X) > 0 && residual.norm() < std::max(4.0, acThreshold))
                             {
                                 const double scale = (_params.featureConstraint == EFeatureConstraint::BASIC) ? 0.0 : featI.scale();
-                                landmark.observations[I] = Observation(xI, track.featPerView.at(I), scale);
+                                landmark.observations[I] = Observation(xI, track.featPerView.at(I).featureId, scale);
                                 ++extented_track;
                             }
                         }
@@ -2034,7 +2034,7 @@ void ReconstructionEngine_sequentialSfM::triangulate_2Views(SfMData& scene,
                             if (poseJ.depth(landmark.X) > 0 && residual.norm() < std::max(4.0, acThreshold))
                             {
                                 const double scale = (_params.featureConstraint == EFeatureConstraint::BASIC) ? 0.0 : featJ.scale();
-                                landmark.observations[J] = Observation(xJ, track.featPerView.at(J), scale);
+                                landmark.observations[J] = Observation(xJ, track.featPerView.at(J).featureId, scale);
                                 ++extented_track;
                             }
                         }
@@ -2084,8 +2084,8 @@ void ReconstructionEngine_sequentialSfM::triangulate_2Views(SfMData& scene,
 
                             const double scaleI = (_params.featureConstraint == EFeatureConstraint::BASIC) ? 0.0 : featI.scale();
                             const double scaleJ = (_params.featureConstraint == EFeatureConstraint::BASIC) ? 0.0 : featJ.scale();
-                            landmark.observations[I] = Observation(xI, track.featPerView.at(I), scaleI);
-                            landmark.observations[J] = Observation(xJ, track.featPerView.at(J), scaleJ);
+                            landmark.observations[I] = Observation(xI, track.featPerView.at(I).featureId, scaleI);
+                            landmark.observations[J] = Observation(xJ, track.featPerView.at(J).featureId, scaleJ);
 
                             ++new_added_track;
                         }  // critical

diff --git a/src/aliceVision/sfm/pipeline/structureFromKnownPoses/StructureEstimationFromKnownPoses.cpp b/src/aliceVision/sfm/pipeline/structureFromKnownPoses/StructureEstimationFromKnownPoses.cpp
@@ -213,7 +213,7 @@ void StructureEstimationFromKnownPoses::filter(const SfMData& sfmData, const Pai
                         for (auto iter = subTrack.featPerView.begin(); iter != subTrack.featPerView.end(); ++iter)
                         {
                             const size_t imaIndex = iter->first;
-                            const size_t featIndex = iter->second;
+                            const size_t featIndex = iter->second.featureId;
                             const View* view = sfmData.getViews().at(imaIndex).get();
 
                             std::shared_ptr<camera::IntrinsicBase> cam = sfmData.getIntrinsics().at(view->getIntrinsicId());
@@ -239,9 +239,9 @@ void StructureEstimationFromKnownPoses::filter(const SfMData& sfmData, const Pai
                                 std::advance(iterJ, 1);
                                 std::advance(iterK, 2);
 
-                                _tripletMatches[std::make_pair(I, J)][subTrack.descType].emplace_back(iterI->second, iterJ->second);
-                                _tripletMatches[std::make_pair(J, K)][subTrack.descType].emplace_back(iterJ->second, iterK->second);
-                                _tripletMatches[std::make_pair(I, K)][subTrack.descType].emplace_back(iterI->second, iterK->second);
+                                _tripletMatches[std::make_pair(I, J)][subTrack.descType].emplace_back(iterI->second.featureId, iterJ->second.featureId);
+                                _tripletMatches[std::make_pair(J, K)][subTrack.descType].emplace_back(iterJ->second.featureId, iterK->second.featureId);
+                                _tripletMatches[std::make_pair(I, K)][subTrack.descType].emplace_back(iterI->second.featureId, iterK->second.featureId);
                             }
                         }
                     }
@@ -279,7 +279,7 @@ void StructureEstimationFromKnownPoses::triangulate(SfMData& sfmData,
         for (auto it = track.featPerView.begin(); it != track.featPerView.end(); ++it)
         {
             const size_t imaIndex = it->first;
-            const size_t featIndex = it->second;
+            const size_t featIndex = it->second.featureId;
             const feature::Regions& regions = regionsPerView.getRegions(imaIndex, track.descType);
             const feature::PointFeature& feat = regions.Features()[featIndex];
 

diff --git a/src/aliceVision/track/Track.hpp b/src/aliceVision/track/Track.hpp
@@ -56,14 +56,19 @@ inline std::ostream& operator<<(std::ostream& os, const KeypointId& k)
     return os;
 }
 
+struct TrackItem
+{
+    std::size_t featureId;
+};
+
 /**
  * @brief A Track is a feature visible accross multiple views.
  * Tracks are generated by the fusion of all matches accross all images.
  */
 struct Track
 {
     /// Data structure to store a track: collection of {ViewId, FeatureId}
-    using FeatureIdPerView = stl::flat_map<std::size_t, std::size_t>;
+    using FeatureIdPerView = stl::flat_map<std::size_t, TrackItem>;
 
     Track() {}
 

diff --git a/src/aliceVision/track/TracksBuilder.cpp b/src/aliceVision/track/TracksBuilder.cpp
@@ -186,7 +186,7 @@ void TracksBuilder::exportToSTL(TracksMap& allTracks) const
             const IndexedFeaturePair& currentPair = _d->map_nodeToIndex.at(iit);
             // all descType inside the track will be the same
             outTrack.descType = currentPair.second.descType;
-            outTrack.featPerView[currentPair.first] = currentPair.second.featIndex;
+            outTrack.featPerView[currentPair.first].featureId = currentPair.second.featIndex;
         }
     }
 }

diff --git a/src/aliceVision/track/trackIO.cpp b/src/aliceVision/track/trackIO.cpp
@@ -9,6 +9,21 @@
 namespace aliceVision {
 namespace track {
 
+void tag_invoke(const boost::json::value_from_tag&, boost::json::value& jv, aliceVision::track::TrackItem const& input)
+{
+    jv = {{"featureId", boost::json::value_from(input.featureId)}};
+}
+
+aliceVision::track::TrackItem tag_invoke(boost::json::value_to_tag<aliceVision::track::TrackItem>, boost::json::value const& jv)
+{
+    const boost::json::object& obj = jv.as_object();
+
+    aliceVision::track::TrackItem ret;
+    ret.featureId = boost::json::value_to<std::size_t>(obj.at("featureId"));
+
+    return ret;
+}
+
 void tag_invoke(const boost::json::value_from_tag&, boost::json::value& jv, aliceVision::track::Track const& input)
 {
     jv = {{"descType", EImageDescriberType_enumToString(input.descType)}, {"featPerView", boost::json::value_from(input.featPerView)}};
@@ -20,7 +35,7 @@ aliceVision::track::Track tag_invoke(boost::json::value_to_tag<aliceVision::trac
 
     aliceVision::track::Track ret;
     ret.descType = feature::EImageDescriberType_stringToEnum(boost::json::value_to<std::string>(obj.at("descType")));
-    ret.featPerView = flat_map_value_to<size_t>(obj.at("featPerView"));
+    ret.featPerView = flat_map_value_to<track::TrackItem>(obj.at("featPerView"));
 
     return ret;
 }

diff --git a/src/aliceVision/track/track_test.cpp b/src/aliceVision/track/track_test.cpp
@@ -77,7 +77,8 @@ BOOST_AUTO_TEST_CASE(Track_Simple)
         BOOST_CHECK_EQUAL(i, iterT->first);
         for (auto iter = iterT->second.featPerView.begin(); iter != iterT->second.featPerView.end(); ++iter)
         {
-            BOOST_CHECK(GT_Tracks[cpt] == std::make_pair(iter->first, iter->second));
+            BOOST_CHECK(GT_Tracks[cpt].first == iter->first);
+            BOOST_CHECK(GT_Tracks[cpt].second == iter->second.featureId);
             ++cpt;
         }
     }
@@ -156,7 +157,13 @@ BOOST_AUTO_TEST_CASE(Track_Conflict)
     // 0, {(0,0) (1,0) (2,0)}
     // 1, {(0,1) (1,1) (2,6)}
     const std::pair<std::size_t, std::size_t> GT_Tracks[] = {
-      std::make_pair(0, 0), std::make_pair(1, 0), std::make_pair(2, 0), std::make_pair(0, 1), std::make_pair(1, 1), std::make_pair(2, 6)};
+      std::make_pair(0, 0), 
+      std::make_pair(1, 0), 
+      std::make_pair(2, 0), 
+      std::make_pair(0, 1), 
+      std::make_pair(1, 1), 
+      std::make_pair(2, 6)
+    };
 
     BOOST_CHECK_EQUAL(2, map_tracks.size());
     std::size_t cpt = 0, i = 0;
@@ -165,7 +172,8 @@ BOOST_AUTO_TEST_CASE(Track_Conflict)
         BOOST_CHECK_EQUAL(i, iterT->first);
         for (auto iter = iterT->second.featPerView.begin(); iter != iterT->second.featPerView.end(); ++iter)
         {
-            BOOST_CHECK(GT_Tracks[cpt] == std::make_pair(iter->first, iter->second));
+            BOOST_CHECK(GT_Tracks[cpt].first == iter->first);
+            BOOST_CHECK(GT_Tracks[cpt].second == iter->second.featureId);
             ++cpt;
         }
     }

diff --git a/src/aliceVision/track/tracksUtils.cpp b/src/aliceVision/track/tracksUtils.cpp
@@ -199,7 +199,7 @@ bool getFeatureIdInViewPerTrack(const TracksMap& allTracks, const std::set<std::
         const Track& map_ref = iterT->second;
         auto iterSearch = map_ref.featPerView.find(viewId);
         if (iterSearch != map_ref.featPerView.end())
-            out_featId->emplace_back(map_ref.descType, iterSearch->second);
+            out_featId->emplace_back(map_ref.descType, iterSearch->second.featureId);
     }
     return !out_featId->empty();
 }
@@ -220,8 +220,8 @@ void tracksToIndexedMatches(const TracksMap& tracks, const std::vector<IndexT>&
         // check we have 2 elements for a track.
         assert(map_ref.featPerView.size() == 2);
 
-        const IndexT indexI = (map_ref.featPerView.begin())->second;
-        const IndexT indexJ = (++map_ref.featPerView.begin())->second;
+        const IndexT indexI = (map_ref.featPerView.begin())->second.featureId;
+        const IndexT indexJ = (++map_ref.featPerView.begin())->second.featureId;
 
         vec_indexref.emplace_back(indexI, indexJ);
     }

diff --git a/src/software/export/main_exportTracks.cpp b/src/software/export/main_exportTracks.cpp
@@ -187,9 +187,9 @@ int aliceVision_main(int argc, char ** argv)
           const PointFeatures& featuresI = featuresPerView.getFeatures(viewI->getViewId(), descType);
           const PointFeatures& featuresJ = featuresPerView.getFeatures(viewJ->getViewId(), descType);
 
-          const PointFeature& imaA = featuresI[obsIt->second];
+          const PointFeature& imaA = featuresI[obsIt->second.featureId];
           ++obsIt;
-          const PointFeature& imaB = featuresJ[obsIt->second];
+          const PointFeature& imaB = featuresJ[obsIt->second.featureId];
 
           svgStream.drawLine(imaA.x(), imaA.y(), imaB.x()+dimImageI.first, imaB.y(), svgStyle().stroke("green", 2.0));
         }
@@ -205,9 +205,9 @@ int aliceVision_main(int argc, char ** argv)
           const PointFeatures& featuresI = featuresPerView.getFeatures(viewI->getViewId(), descType);
           const PointFeatures& featuresJ = featuresPerView.getFeatures(viewJ->getViewId(), descType);
 
-          const PointFeature& imaA = featuresI[obsIt->second];
+          const PointFeature& imaA = featuresI[obsIt->second.featureId];
           ++obsIt;
-          const PointFeature& imaB = featuresJ[obsIt->second];
+          const PointFeature& imaB = featuresJ[obsIt->second.featureId];
 
           const std::string featColor = describerTypeColor(descType);