Merge pull request #4 from neatdecisions/v0.2.2

V0.2.2

.gitignore

@@ -41,4 +41,5 @@
 build
 dist
 Debug
+DebugOptimized
 Release

README.md

@@ -1,6 +1,6 @@
 # Detwinner
 
-Detwinner is a tool for GNOME environment which allows searching and removing duplicate files and similar images.
+Detwinner is a tool for the Linux desktop which allows searching and removing duplicate files and similar images.
 
 ## Main functionalities
 

configure.sh

@@ -1,10 +1,12 @@
 rm -rf Debug
 rm -rf Release
+rm -rf DebugOptimized
 mkdir Debug
+mkdir DebugOptimized
 mkdir Release
 cd Debug
-#CC=clang CXX=clang++ meson .. --buildtype=debug
 meson .. --buildtype=debug
 cd ../Release
-#CC=clang CXX=clang++ meson .. --buildtype=release
 meson .. --buildtype=release
+cd ../DebugOptimized
+meson .. --buildtype=debugoptimized

data/com.neatdecisions.Detwinner.appdata.xml.in

@@ -6,6 +6,7 @@
   <project_license>GPL-3.0+</project_license>
   <name>Detwinner</name>
   <summary>Find and remove duplicate files and similar images</summary>
+  <developer_name>Neat Decisions</developer_name>
   <description>
     <p>Detwinner is a tool to search and remove exact file duplicates and similar images.</p>
     <p>You can search for either exact duplicates (100% byte-to-byte match is ensured by the special hash function) or similar images (Detwinner generates groups of images that look alike using similarity level specified by you). You can also narrow your search by specifying file size, attributes and regular expressions.</p>
@@ -26,12 +27,23 @@
   </screenshots>
   <url type="homepage">https://neatdecisions.com/products/detwinner-linux/</url>
   <url type="bugtracker">https://github.com/neatdecisions/detwinner/issues</url>
+  <url type="donation">https://neatdecisions.com/products/detwinner-linux/donate.php</url>
   <update_contact>[email protected]</update_contact>
   ​<translation type="gettext">com.neatdecisions.Detwinner</translation>
   <provides>
     <binary>detwinner</binary>
   </provides>
   <releases>
+    <release version="0.2.2" date="2019-09-08">
+      <description>
+        <p>Main changes in this version:</p>
+        <ul>
+          <li>Ignore symlinks when comparing files</li>
+          <li>Properly resolve file deletion status</li>
+          <li>Overall performance improvements</li>
+        </ul>
+      </description>
+    </release>
     <release version="0.2.1" date="2019-09-03">
       <description>
         <p>Minor release which only includes changes in the AppStream metadata.</p>

data/detwinner.1

@@ -1,6 +1,6 @@
-.TH detwinner 1 "03 September 2019" "version 0.2.1"
+.TH detwinner 1 "08 September 2019" "version 0.2.2"
 .SH NAME
-Detwinner - duplicate file finder for GNOME environment
+Detwinner - duplicate file finder for the Linux desktop
 .SH SYNOPSIS
 detwinner
 .SH DESCRIPTION

meson.build

@@ -1,5 +1,5 @@
 project('detwinner', ['c', 'cpp'],
-  version: '0.2.1',
+  version: '0.2.2',
   license: 'GPL3+',
   meson_version: '>=0.45.0',
 	default_options : ['c_std=c11', 'cpp_std=c++17', 'warning_level=3', 'werror=true'])

src/detwinner-lib/logic/CommonDataTypes.hpp

@@ -3,14 +3,15 @@
  Name        : CommonDataTypes.hpp
  Author      : NeatDecisions
  Version     :
- Copyright   : Copyright © 2018 Neat Decisions. All rights reserved.
+ Copyright   : Copyright © 2018–2019 Neat Decisions. All rights reserved.
  Description : Detwinner
  ===============================================================================
  */
 
 #ifndef LOGIC_COMMONDATATYPES_HPP_
 #define LOGIC_COMMONDATATYPES_HPP_
 
+#include <string>
 #include <vector>
 #include <compat/optional.hpp>
 
@@ -27,7 +28,6 @@ struct DuplicateContainer
 		{
 			unsigned int width = 0;
 			unsigned int height = 0;
-			ImageSize_t() = default;
 			ImageSize_t(unsigned int w, unsigned int h) : width(w), height(h){}
 		};
 
@@ -38,10 +38,8 @@ struct DuplicateContainer
 			size(size), name(fileName) {}
 
 		FileDataInfo(unsigned long long size, const std::string & fileName, unsigned int width, unsigned int height) :
-			size(size), name(fileName), imageResolution(stdx::make_optional(ImageSize_t(width, height))) {}
+			size(size), name(fileName), imageResolution(stdx::make_optional<ImageSize_t>(width, height)) {}
 	};
-	DuplicateContainer() = default;
-	explicit DuplicateContainer(const std::vector<FileDataInfo> & files) : files(files) {}
 	std::vector<FileDataInfo> files;
 };
 

src/detwinner-lib/logic/FileIndexer.cpp

@@ -3,7 +3,7 @@
  Name        : FileIndexer.cpp
  Author      : NeatDecisions
  Version     :
- Copyright   : Copyright © 2018 Neat Decisions. All rights reserved.
+ Copyright   : Copyright © 2018–2019 Neat Decisions. All rights reserved.
  Description : Detwinner
  ===============================================================================
  */
@@ -158,12 +158,12 @@ FileIndexer::getFileType(const std::string & filePath) const
 
 	if (!m_settings.searchHiddenFiles)
 	{
-		const fs::path fp(filePath);
-		if (fp.filename().string().find(".") == 0) return FileType_t::kIgnored;
+		const std::string fileName = fs::path(filePath).filename().string();
+		if (!fileName.empty() && (fileName.front() == '.')) return FileType_t::kIgnored;
 	}
 
 	std::error_code errorCode;
-	const fs::file_status fileStatus = fs::status(filePath, errorCode);
+	const fs::file_status fileStatus = fs::symlink_status(filePath, errorCode);
 	if (errorCode) return FileType_t::kUnknown;
 
 	if (fs::is_directory(fileStatus)) return FileType_t::kDirectory;
@@ -193,21 +193,18 @@ FileIndexer::getFileType(const std::string & filePath) const
 			}
 		}
 
-		const unsigned long long fileSize = fs::file_size(filePath, errorCode);
-		if (errorCode) return FileType_t::kUnknown;
-		if ( m_settings.maxFileSize && (m_settings.maxFileSize.value() <= fileSize) ) return FileType_t::kIgnored;
-		if ( m_settings.minFileSize && (m_settings.minFileSize.value() >= fileSize) ) return FileType_t::kIgnored;
-
-		bool regexMatched = m_includedRegexps.empty();
-		for (auto && regex: m_includedRegexps)
+		if (m_settings.maxFileSize || m_settings.minFileSize)
 		{
-			if (std::regex_match(filePath, regex))
-			{
-				regexMatched = true;
-				break;
-			}
+			const unsigned long long fileSize = fs::file_size(filePath, errorCode);
+			if (errorCode) return FileType_t::kUnknown;
+			if ( m_settings.maxFileSize && (m_settings.maxFileSize.value() <= fileSize) ) return FileType_t::kIgnored;
+			if ( m_settings.minFileSize && (m_settings.minFileSize.value() >= fileSize) ) return FileType_t::kIgnored;
 		}
+
+		const bool regexMatched = m_includedRegexps.empty() || std::any_of(m_includedRegexps.begin(), m_includedRegexps.end(),
+				[&filePath](const auto & r) { return std::regex_match(filePath, r); });
 		if (!regexMatched) return FileType_t::kIgnored;
+
 		return FileType_t::kRegular;
 	}
 	return FileType_t::kUnknown;

src/detwinner-lib/logic/images/HistogramT.hpp

@@ -13,7 +13,7 @@
 
 
 #include <array>
-#include <algorithm>
+#include <numeric>
 
 
 namespace detwinner {
@@ -32,24 +32,25 @@ class HistogramT
 	using BinValue_t = uint_least16_t;
 	static constexpr std::size_t kBinCount = BinNumber;
 
-	float compare(const HistogramT & hist) const
+	float compare(const HistogramT & hist) const noexcept
 	{
 		// Hassanat distance
-		float num = 0.0f;
-		for (size_t i = 0; i < BinNumber; ++i)
-		{
-			const auto & mm = std::minmax(bins[i], hist.bins[i]);
-			num += ( 1.0f - (1.0f + mm.first) / (1.0f + mm.second) );
-		}
-		return num / (BinNumber - 1);
+		constexpr auto aLbmAccumulator = [](float a, float b) { return a + b; };
+		constexpr auto aLbmTransformer = [](BinValue_t a, BinValue_t b) {
+			return ( a < b ? (1.0f + a) / (1.0f + b) :
+			                 (1.0f + b) / (1.0f + a) );
+		};
+
+		return (BinNumber - std::inner_product(bins.begin(), bins.end(), hist.bins.begin(), 0.0f,
+				aLbmAccumulator, aLbmTransformer)) / (BinNumber - 1);
 	}
 
-	BinValue_t getBinValue(const std::size_t binNumber) const
+	BinValue_t getBinValue(const std::size_t binNumber) const noexcept
 	{
 		return (binNumber < BinNumber) ? bins[binNumber] : 0;
 	}
 
-	void setBinValue(const std::size_t binNumber, const BinValue_t value)
+	void setBinValue(const std::size_t binNumber, const BinValue_t value) noexcept
 	{
 		if (binNumber < BinNumber) bins[binNumber] = value;
 	}

src/detwinner-lib/logic/images/ImageFeatures.cpp

@@ -10,12 +10,11 @@
 
 #include <logic/images/ImageFeatures.hpp>
 
-#include <assert.h>
+#include <algorithm>
 #include <cmath>
 #include <limits>
-#include <vector>
 #include <numeric>
-
+#include <vector>
 
 namespace detwinner {
 namespace logic {
@@ -28,15 +27,6 @@ ImageFeatures::ImageFeatures(const unsigned int id, const float aspect) :
 {}
 
 
-//------------------------------------------------------------------------------
-template <class Histogram_t>
-float
-ImageFeatures::compareHistogram(const Histogram_t & h1, const Histogram_t & h2) const
-{
-	return h1.compare(h2);
-}
-
-
 //------------------------------------------------------------------------------
 float
 ImageFeatures::compare(const ImageFeatures & f, bool processRotations) const
@@ -59,10 +49,10 @@ ImageFeatures::compare(const ImageFeatures & f, bool processRotations) const
 		for (auto sectionIndex = 0; sectionIndex < kSectionCount; ++sectionIndex)
 		{
 			const std::size_t sectionIndex2 = (i + sectionIndex) % kSectionCount;
-			values[0] += compareHistogram(histY[sectionIndex], f.histY[sectionIndex2]);
-			values[1] += compareHistogram(histU[sectionIndex], f.histU[sectionIndex2]);
-			values[2] += compareHistogram(histV[sectionIndex], f.histV[sectionIndex2]);
-			values[3] += compareHistogram(histI[sectionIndex], f.histI[sectionIndex2]);
+			values[0] += histY[sectionIndex].compare(f.histY[sectionIndex2]);
+			values[1] += histU[sectionIndex].compare(f.histU[sectionIndex2]);
+			values[2] += histV[sectionIndex].compare(f.histV[sectionIndex2]);
+			values[3] += histI[sectionIndex].compare(f.histI[sectionIndex2]);
 		}
 
 		std::transform(values.begin(), values.end(), values.begin(), [](float v) { return v / 4.0f; } );

src/detwinner-lib/logic/images/ImageFeatures.hpp

@@ -33,7 +33,7 @@ struct ImageFeatures
 {
 	ImageFeatures(const unsigned int id, const float aspect);
 
-	unsigned int getId() const { return id; }
+	unsigned int getId() const noexcept { return id; }
 
 	static constexpr uint8_t kSectionCount = 4;
 	Histogram histY[kSectionCount];
@@ -44,9 +44,6 @@ struct ImageFeatures
 	float compare(const ImageFeatures & f, bool processRotations) const;
 
 private:
-	template <class Histogram_t>
-	float compareHistogram(const Histogram_t & h1, const Histogram_t & h2) const;
-
 	unsigned int id;
 	float aspect;
 };

src/detwinner-lib/logic/images/ImageFeaturesBridge.cpp

@@ -23,7 +23,7 @@ template <std::size_t BinCount>
 void
 ImageFeaturesBridge::NormalizeHistogramFromArrayT(
 	const std::array<std::size_t, BinCount> & realHist,
-	HistogramT<BinCount> & hist)
+	HistogramT<BinCount> & hist) noexcept
 {
 	for (std::size_t i = 0; i < BinCount; ++i)
 	{
@@ -48,7 +48,7 @@ ImageFeaturesBridge::GetIntensityHistogram(
 	}
 
 	constexpr int kBinNumber = HistogramI::kBinCount;
-	constexpr float kBinSize = 255.0 / kBinNumber;
+	constexpr float kBinSize = 255.0f / kBinNumber;
 
 	std::array<std::size_t, kBinNumber> realHistI{};
 
@@ -107,7 +107,7 @@ ImageFeaturesBridge::GetYUVHistograms(
 		for (unsigned int y = roi.yOff(); y < height; ++y)
 		{
 			const Magick::ColorYUV & color = image.pixelColor(x, y);
-			const double yyy = ( color.alpha() > 0.8 ) ? 1.0 : color.y();
+			const float yyy = ( color.alpha() > 0.8 ) ? 1.0 : color.y();
 			int n = static_cast<int>(floor(( yyy - yMin ) / yBinSize));
 			if (n >= kBinNumber) n = kBinNumber - 1;
 			if (n < 0) n = 0;
@@ -146,16 +146,18 @@ ImageFeaturesBridge::GetImageFeatures(Magick::Image & image, unsigned int id)
 	constexpr unsigned int kImageDimension = 128;
 	static const Magick::Geometry kResizeGeometry(kImageDimension, kImageDimension);
 
-	ImageFeatures feats(id, static_cast<float>(image.size().height()) / static_cast<float>(image.size().width()));
+	Magick::Geometry imageSize = image.size();
+	ImageFeatures feats(id, static_cast<float>(imageSize.height()) / static_cast<float>(imageSize.width()));
 
-	if (image.size().height() > kImageDimension || image.size().width() > kImageDimension)
+	if (imageSize.height() > kImageDimension || imageSize.width() > kImageDimension)
 	{
 		image.thumbnail(kResizeGeometry);
+		imageSize = image.size();
 	}
 
-	const unsigned int dw = image.size().width() / 2;
+	const unsigned int dw = imageSize.width() / 2;
 	const unsigned int dw1 = dw + 2 * dw % 2;
-	const unsigned int dh = image.size().height() / 2;
+	const unsigned int dh = imageSize.height() / 2;
 	const unsigned int dh1 = dh + 2 * dh % 2;
 
 	GetYUVHistograms(image, Magick::Geometry(dw, dh, 0, 0), feats.histY[0], feats.histU[0], feats.histV[0]);

src/detwinner-lib/logic/images/ImageFeaturesBridge.hpp

@@ -35,7 +35,7 @@ class ImageFeaturesBridge
 	template <std::size_t BinCount>
 	static void NormalizeHistogramFromArrayT(
 		const std::array<std::size_t, BinCount> & realHist,
-		HistogramT<BinCount> & hist);
+		HistogramT<BinCount> & hist) noexcept;
 
 	static void GetIntensityHistogram(
 			const Magick::Image & image,

src/detwinner-lib/logic/images/ImageFeaturesBuilder.cpp

@@ -20,8 +20,9 @@ namespace logic {
 namespace images {
 
 
-constexpr unsigned int kMinimumFilesPerThread = 1000;
-
+namespace {
+	constexpr unsigned int kMinimumFilesPerThread = 1000;
+}
 
 //------------------------------------------------------------------------------
 ImageFeaturesBuilder::ImageFeaturesBuilder(
@@ -96,7 +97,7 @@ ImageFeaturesBuilder::executeInternal(const std::size_t startIndex, const std::s
 	{
 		try
 		{
-			img.read(kResizeGeometry, m_fileNames.at(i));
+			img.read(kResizeGeometry, m_fileNames[i]);
 			imageFeatures.push_back(ImageFeaturesBridge::GetImageFeatures(img, i));
 		} catch (const Magick::Exception&)
 		{