diff --git a/CMakeLists.txt b/CMakeLists.txt index ac8c651..27bdc77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -230,6 +230,20 @@ if (EMSCRIPTEN) configure_file(${CMAKE_SOURCE_DIR}/index-${TARGET}-tmpl.html ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/index.html @ONLY) configure_file(${CMAKE_SOURCE_DIR}/style.css ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/style.css @ONLY) + + # keytap3-gui + set(TARGET keytap3-gui) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${TARGET}) + + set_target_properties(${TARGET} PROPERTIES LINK_FLAGS " \ + -s TOTAL_MEMORY=536870912 \ + -s FORCE_FILESYSTEM=1 \ + -s LZ4=1 \ + --preload-file ${PROJECT_SOURCE_DIR}/data/ggwords-6-gram.dat.binary@/data/ \ + ") + + configure_file(${CMAKE_SOURCE_DIR}/index-${TARGET}-tmpl.html ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/index.html @ONLY) + configure_file(${CMAKE_SOURCE_DIR}/style.css ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/style.css @ONLY) endif() if (NOT EMSCRIPTEN) diff --git a/README.md b/README.md index ed0f08c..98ae5b4 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ A more detailed description of the tool is available here: [Keytap2 discussion]( [CTF: can you guess the text being typed?](https://ggerganov.github.io/keytap-challenge/) [Try it online:](https://keytap2.ggerganov.com) - + ### Keytap3 @@ -39,6 +39,8 @@ automated and does not require any manual intervation during the text recovery p [Video: short demo of using Keytap3](https://youtu.be/5aphvxpSt3o) +[GUI for Keytap3](https://keytap3.ggerganov.com) + [Check if your keyboard is vulnerable to Keytap:](https://keytap3.ggerganov.com) @@ -99,6 +101,7 @@ Short summary of the available tools. If the status of the tool is not **stable* | **keytap-gui** | gui | **stable** | | **keytap2-gui** | gui | **stable** | | **keytap3** | text | **stable** | +| **keytap3-gui** | gui | **stable** | | - | *extra* | - | | **guess-qp** | text | experiment | | **guess-qp2** | text | experiment | @@ -107,7 +110,6 @@ Short summary of the available tools. If the status of the tool is not **stable* | **subreak** | text | experiment | | **key-average-gui** | gui | experiment | | **keytap2** | text | experiment | -| **keytap3-gui** | gui | experiment | ## Tool details @@ -157,6 +159,8 @@ Short summary of the available tools. If the status of the tool is not **stable* ./keytap-gui input0.kbd [input1.kbd] [input2.kbd] ... [-cN] [-CN] + Online demo: https://keytap.ggerganov.com + --- * **keytap2-gui** record.kbd n-gram-dir [-pN] [-cN] [-CN] @@ -165,6 +169,8 @@ Short summary of the available tools. If the status of the tool is not **stable* ./keytap2-gui record.kbd ../data + Online demo: https://keytap2.ggerganov.com + --- * **keytap3** @@ -177,6 +183,16 @@ Short summary of the available tools. If the status of the tool is not **stable* --- +* **keytap3-gui** + + GUI version of the **keytap3** tool. + + ./keytap3-gui input.kbd ../data [-cN] [-CN] [-pF] [-tF] [-FN] [-fN] + + Online demo: https://keytap3-gui.ggerganov.com + + --- + * **view-full-gui** Visualize waveforms recorded with the **record-full** tool. Can also playback the audio data. diff --git a/index-keytap2-gui-tmpl.html b/index-keytap2-gui-tmpl.html index 82ebd1a..502e1a8 100644 --- a/index-keytap2-gui-tmpl.html +++ b/index-keytap2-gui-tmpl.html @@ -269,6 +269,6 @@

Keytap2 - acoustic keyboard eavesdropping based on language n-gram frequenci - + diff --git a/index-keytap3-gui-tmpl.html b/index-keytap3-gui-tmpl.html new file mode 100644 index 0000000..640308e --- /dev/null +++ b/index-keytap3-gui-tmpl.html @@ -0,0 +1,253 @@ + + + + + Keytap3 + + + + + + + + + + + + + + + + + + + + +
+
+

Keytap3 - acoustic keyboard eavesdropping

+ + + This is a GUI for Keytap3
+ +
+ + This tool runs entirely in your browser. No data is sent or stored to a server. + +

+ + Press the "Init" button to start: + +

+ + +
+ +

+ +
+
+ +
+ + +
+ View on GitHub + + +
+ + + + + + diff --git a/keytap3-gui.cpp b/keytap3-gui.cpp index fb7c896..79d5bb4 100644 --- a/keytap3-gui.cpp +++ b/keytap3-gui.cpp @@ -92,19 +92,22 @@ struct stParameters { int32_t offsetFromPeak_samples = kKeyWidth_samples - kKeyOffset_samples; int32_t alignWindow_samples = kKeyAlign_samples; - std::vector valuesClusters = { 40, 50, 60, 70, }; - std::vector valuesWEnglishFreq = { 1, 2, 5, 10, }; + float wEnglish = 30.0f; + int nHypothesesToKeep = 500; + + std::vector valuesClusters = { 40, 60, 80, 100, }; + std::vector valuesFSpread = { 0.5f, 0.75f, 1.0f, 1.5f, 2.0f, }; int32_t nProcessors() const { - return valuesClusters.size()*valuesWEnglishFreq.size(); + return valuesClusters.size()*valuesFSpread.size(); } int32_t valueForProcessorClusters(int idx) const { - return valuesClusters[idx/(valuesWEnglishFreq.size())]; + return valuesClusters[idx/(valuesFSpread.size())]; } - float valueForProcessorWEnglishFreq(int idx) const { - return valuesWEnglishFreq[idx%valuesWEnglishFreq.size()]; + float valueForProcessorFSpread(int idx) const { + return valuesFSpread[idx%valuesFSpread.size()]; } Cipher::TParameters cipher; @@ -117,7 +120,7 @@ struct stStateUI { bool recalculateSimilarityMap = false; bool resetOptimization = false; bool changeProcessing = false; - bool applyClusters = false; + bool applyParameters = false; bool applyHints = false; void clear() { memset(this, 0, sizeof(Flags)); } @@ -161,6 +164,7 @@ struct stStateUI { bool openParametersWindow = false; bool loadRecord = false; bool loadKeyPresses = false; + bool findBestCutoffFreq = false; bool rescaleWaveform = true; int outputRecordId = 0; @@ -264,7 +268,7 @@ template<> bool TripleBuffer::update(bool force) { buffer.processing = this->processing; } - if (this->flags.applyClusters) { + if (this->flags.applyParameters) { buffer.params = this->params; } @@ -391,7 +395,7 @@ bool renderKeyPresses(stStateUI & stateUI, const TWaveform & waveform, TKeyPress } } -#ifndef __EMSCRIPTEN__ +#if 0 if ((int) keyPresses.size() >= lastKeyPresses + 10) { lastKeyPresses = keyPresses.size(); @@ -683,6 +687,7 @@ bool renderKeyPresses(stStateUI & stateUI, const TWaveform & waveform, TKeyPress stateUI.recalculateKeyPresses = true; stateUI.lastSize = stateUI.lastSize - 1; stateUI.recording = false; + stateUI.findBestCutoffFreq = true; stateUI.rescaleWaveform = true; } { @@ -720,7 +725,7 @@ bool renderKeyPresses(stStateUI & stateUI, const TWaveform & waveform, TKeyPress } bool renderResults(stStateUI & stateUI) { - float curHeight = std::min(g_windowSizeY - stateUI.windowHeightTitleBar - stateUI.windowHeightKeyPesses, (12 + stateUI.results.size()*kTopResultsPerProcessor)*ImGui::GetTextLineHeightWithSpacing()); + float curHeight = std::min(g_windowSizeY - stateUI.windowHeightTitleBar - stateUI.windowHeightKeyPesses, (12 + std::min(8lu, stateUI.results.size())*kTopResultsPerProcessor)*ImGui::GetTextLineHeightWithSpacing()); ImGui::SetNextWindowPos(ImVec2(0, stateUI.windowHeightTitleBar + stateUI.windowHeightKeyPesses), ImGuiCond_Always); ImGui::SetNextWindowSize(ImVec2(1.0f*g_windowSizeX, curHeight), ImGuiCond_Always); if (ImGui::Begin("Results", nullptr, ImGuiWindowFlags_HorizontalScrollbar | ImGuiWindowFlags_NoMove)) { @@ -796,20 +801,31 @@ bool renderResults(stStateUI & stateUI) { ImGui::PushItemWidth(200.0); - if (ImGui::SliderInt("Clusters", &stateUI.params.valuesClusters[0], 30, 150.0)) { - stateUI.flags.applyClusters = true; - stateUI.doUpdate = true; - } + //if (ImGui::SliderInt("Clusters", &stateUI.params.valuesClusters[0], 30, 150.0)) { + // stateUI.flags.applyParameters = true; + // stateUI.doUpdate = true; + //} + + //if (ImGui::SliderFloat("Spread", &stateUI.params.fSpread, 0.1f, 3.0f)) { + // stateUI.flags.applyParameters = true; + // stateUI.doUpdate = true; + //} ImGui::SameLine(); - if (ImGui::Checkbox("Auto", &stateUI.autoHint)) { - } - if (ImGui::IsItemHovered()) { - ImGui::BeginTooltip(); - ImGui::Text("If checked, hints will be updated automatically based on the suggestions"); - ImGui::EndTooltip(); + if (ImGui::SliderInt("Keep", &stateUI.params.nHypothesesToKeep, 20, 2000)) { + stateUI.flags.applyParameters = true; + stateUI.doUpdate = true; } + //ImGui::SameLine(); + //if (ImGui::Checkbox("Auto", &stateUI.autoHint)) { + //} + //if (ImGui::IsItemHovered()) { + // ImGui::BeginTooltip(); + // ImGui::Text("If checked, hints will be updated automatically based on the suggestions"); + // ImGui::EndTooltip(); + //} + if (stateUI.results.size() > 0) { int nPerAutoHint = 10; auto id0 = stateUI.results.begin()->second.id%stateUI.params.cipher.nIters; @@ -1243,7 +1259,7 @@ int main(int argc, char ** argv) { const int captureId = argm.count("c") == 0 ? 0 : std::stoi(argm.at("c")); const int nChannels = argm.count("C") == 0 ? 0 : std::stoi(argm.at("C")); const int filterId = argm.count("F") == 0 ? EAudioFilter::FirstOrderHighPass : std::stoi(argm.at("F")); - const int freqCutoff_Hz = argm.count("f") == 0 ? kFreqCutoff_Hz : std::stoi(argm.at("f")); + const int freqCutoff_Hz = argm.count("f") == 0 ? 0 : std::stoi(argm.at("f")); stateUI.params.playbackId = playbackId; stateUI.fnameRecord = argv[1]; @@ -1252,7 +1268,7 @@ int main(int argc, char ** argv) { stateUI.waveformInput.reserve(kSamplesPerFrame*kMaxRecordSize_frames); stateUI.waveformOriginal.reserve(kSamplesPerFrame*kMaxRecordSize_frames); - if (SDL_Init(SDL_INIT_VIDEO|SDL_INIT_TIMER) != 0) { + if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_TIMER) != 0) { printf("Error: %s\n", SDL_GetError()); return -2; } @@ -1282,7 +1298,7 @@ int main(int argc, char ** argv) { parameters.captureId = captureId; parameters.nChannels = nChannels; parameters.sampleRate = kSampleRate; - parameters.filter = (EAudioFilter) filterId; + parameters.filter = EAudioFilter::None; parameters.freqCutoff_Hz = kFreqCutoff_Hz; if (audioLogger.install(std::move(parameters)) == false) { @@ -1312,8 +1328,18 @@ int main(int argc, char ** argv) { printf("Specified file '%s' does not exist\n", argv[1]); //return -4; } else { - printf("[+] Filtering waveform with filter type = %d and cutoff frequency = %d Hz\n", filterId, freqCutoff_Hz); - ::filter(stateUI.waveformOriginal, (EAudioFilter) filterId, freqCutoff_Hz, kSampleRate); + auto freqCutoffCur_Hz = freqCutoff_Hz; + if (freqCutoffCur_Hz == 0) { + const auto tStart = std::chrono::high_resolution_clock::now(); + + freqCutoffCur_Hz = Cipher::findBestCutoffFreq(stateUI.waveformOriginal, (EAudioFilter) filterId, kSampleRate, 100.0f, 1000.0f, 100.0f); + + const auto tEnd = std::chrono::high_resolution_clock::now(); + printf("[+] Found best freqCutoff = %d Hz, took %4.3f seconds\n", freqCutoffCur_Hz, toSeconds(tStart, tEnd)); + } + + printf("[+] Filtering waveform with filter type = %d and cutoff frequency = %d Hz\n", filterId, freqCutoffCur_Hz); + ::filter(stateUI.waveformOriginal, (EAudioFilter) filterId, freqCutoffCur_Hz, kSampleRate); printf("[+] Converting waveform to i16 format ...\n"); if (convert(stateUI.waveformOriginal, stateUI.waveformInput) == false) { @@ -1325,36 +1351,12 @@ int main(int argc, char ** argv) { stateUI.maxSample = calcAbsMax(stateUI.waveformOriginal); } - Cipher::TFreqMap freqMap3; - Cipher::TFreqMap freqMap4; - Cipher::TFreqMap freqMap5; Cipher::TFreqMap freqMap6; - //if (Cipher::loadFreqMap((std::string(argv[2]) + "/./ggwords-3-gram.dat").c_str(), freqMap3) == false) { - // return -5; - //} - //if (Cipher::loadFreqMap((std::string(argv[2]) + "/./ggwords-4-gram.dat").c_str(), freqMap4) == false) { - // return -5; - //} - //if (Cipher::loadFreqMap((std::string(argv[2]) + "/./ggwords-5-gram.dat").c_str(), freqMap5) == false) { - // return -5; - //} - //stateCore.freqMap[0] = &freqMap3; - //stateCore.freqMap[1] = &freqMap4; - //stateCore.freqMap[2] = &freqMap5; - - if (Cipher::loadFreqMapBinary((std::string(argv[2]) + "/./ggwords-3-gram.dat.binary").c_str(), freqMap3) == false) { - return -5; - } - if (Cipher::loadFreqMapBinary((std::string(argv[2]) + "/./ggwords-4-gram.dat.binary").c_str(), freqMap4) == false) { - return -5; - } - if (Cipher::loadFreqMapBinary((std::string(argv[2]) + "/./ggwords-5-gram.dat.binary").c_str(), freqMap5) == false) { - return -5; - } if (Cipher::loadFreqMapBinary((std::string(argv[2]) + "/./ggwords-6-gram.dat.binary").c_str(), freqMap6) == false) { return -5; } + stateCore.freqMap[0] = &freqMap6; stateCore.freqMap[1] = &freqMap6; stateCore.freqMap[2] = &freqMap6; @@ -1410,6 +1412,7 @@ int main(int argc, char ** argv) { } } + recalcSuggestions = false; if (recalcSuggestions) { int n = stateUI.keyPresses.size(); stateUI.suggestions.resize(n); @@ -1437,6 +1440,9 @@ int main(int argc, char ** argv) { stateUI.suggestions[i] = -1; } } + } else { + int n = stateUI.keyPresses.size(); + stateUI.suggestions.resize(n); } } @@ -1604,8 +1610,18 @@ int main(int argc, char ** argv) { if (readFromFile(stateUI.fnameRecord, stateUI.waveformOriginal) == false) { printf("Specified file '%s' does not exist\n", argv[1]); } else { - printf("[+] Filtering waveform with filter type = %d and cutoff frequency = %d Hz\n", filterId, freqCutoff_Hz); - ::filter(stateUI.waveformOriginal, (EAudioFilter) filterId, freqCutoff_Hz, kSampleRate); + auto freqCutoffCur_Hz = freqCutoff_Hz; + if (freqCutoffCur_Hz == 0) { + const auto tStart = std::chrono::high_resolution_clock::now(); + + freqCutoffCur_Hz = Cipher::findBestCutoffFreq(stateUI.waveformOriginal, (EAudioFilter) filterId, kSampleRate, 100.0f, 1000.0f, 100.0f); + + const auto tEnd = std::chrono::high_resolution_clock::now(); + printf("[+] Found best freqCutoff = %d Hz, took %4.3f seconds\n", freqCutoffCur_Hz, toSeconds(tStart, tEnd)); + } + + printf("[+] Filtering waveform with filter type = %d and cutoff frequency = %d Hz\n", filterId, freqCutoffCur_Hz); + ::filter(stateUI.waveformOriginal, (EAudioFilter) filterId, freqCutoffCur_Hz, kSampleRate); printf("[+] Converting waveform to i16 format ...\n"); if (convert(stateUI.waveformOriginal, stateUI.waveformInput) == false) { @@ -1627,6 +1643,23 @@ int main(int argc, char ** argv) { stateUI.loadKeyPresses = false; } + if (stateUI.findBestCutoffFreq) { + auto freqCutoffCur_Hz = freqCutoff_Hz; + if (freqCutoffCur_Hz == 0) { + const auto tStart = std::chrono::high_resolution_clock::now(); + + freqCutoffCur_Hz = Cipher::findBestCutoffFreq(stateUI.waveformOriginal, (EAudioFilter) filterId, kSampleRate, 100.0f, 1000.0f, 100.0f); + + const auto tEnd = std::chrono::high_resolution_clock::now(); + printf("[+] Found best freqCutoff = %d Hz, took %4.3f seconds\n", freqCutoffCur_Hz, toSeconds(tStart, tEnd)); + } + + printf("[+] Filtering waveform with filter type = %d and cutoff frequency = %d Hz\n", filterId, freqCutoffCur_Hz); + ::filter(stateUI.waveformOriginal, (EAudioFilter) filterId, freqCutoffCur_Hz, kSampleRate); + + stateUI.findBestCutoffFreq = false; + } + if (stateUI.rescaleWaveform) { if (convert(stateUI.waveformOriginal, stateUI.waveformInput) == false) { fprintf(stderr, "error : recording failed\n"); @@ -1708,19 +1741,24 @@ int main(int argc, char ** argv) { int n = stateCore.params.nProcessors(); for (int i = 0; i < n; ++i) { - int nClusters = stateCore.params.valueForProcessorClusters(i); - double w = stateCore.params.valueForProcessorWEnglishFreq(i); + const auto nClusters = stateCore.params.valueForProcessorClusters(i); + const auto wEnglish = stateCore.params.wEnglish; + const auto fSpread = stateCore.params.valueForProcessorFSpread(i); + const auto nHypothesesToKeep = stateCore.params.nHypothesesToKeep; Cipher::TParameters params; params.maxClusters = nClusters; - params.wEnglishFreq = w; + params.wEnglishFreq = wEnglish; + params.fSpread = fSpread; + params.nHypothesesToKeep = nHypothesesToKeep; stateCore.processors[i] = Cipher::Processor(); stateCore.processors[i].init( params, *stateCore.freqMap[i%3], stateCore.similarityMap); - printf("[+] Processor %d initialized: cluster = %d, w = %g\n", i, nClusters, w); + printf("[+] Processor %d initialized: cluster = %d, wEnglish = %g, fSpread = %g, nHypothesesToKeep = %d\n", + i, nClusters, wEnglish, fSpread, nHypothesesToKeep); } } @@ -1728,17 +1766,21 @@ int main(int argc, char ** argv) { stateCore.processing = stateUINew.processing; } - if (stateUINew.flags.applyClusters) { + if (stateUINew.flags.applyParameters) { stateCore.params = stateUINew.params; int n = stateCore.params.nProcessors(); for (int i = 0; i < n; ++i) { - int nClusters = stateCore.params.valueForProcessorClusters(i); - double w = stateCore.params.valueForProcessorWEnglishFreq(i); + const auto nClusters = stateCore.params.valueForProcessorClusters(i); + const auto wEnglish = stateCore.params.valueForProcessorFSpread(i); + const auto fSpread = stateCore.params.valueForProcessorFSpread(i); + const auto nHypothesesToKeep = stateCore.params.nHypothesesToKeep; Cipher::TParameters params; params.maxClusters = nClusters; - params.wEnglishFreq = w; + params.wEnglishFreq = wEnglish; + params.fSpread = fSpread; + params.nHypothesesToKeep = nHypothesesToKeep; stateCore.processors[i].init( params, *stateCore.freqMap[i%3], diff --git a/subbreak3.cpp b/subbreak3.cpp index c3e46ed..d222dbb 100644 --- a/subbreak3.cpp +++ b/subbreak3.cpp @@ -703,17 +703,17 @@ namespace Cipher { hypothesesCur.resize(nHypothesesToKeep); hypothesesNew.resize(nHypothesesToKeep*nSymbols); - int nHints = 0; { auto & hcur = hypothesesCur[0]; hcur = { 0.0, getNullCLMap(clusters), {}, std::vector(nSymbols + 1, 0), {}}; - translate(hcur.clMap, clusters, hcur.plain); for (int i = 0; i < (int) params.hint.size(); ++i) { if (params.hint[i] != -1) { - hcur.plain[i] = params.hint[i]; - ++nHints; + if (frand() > 0.5) { + hcur.clMap[clusters[i]] = params.hint[i]; + } } } + translate(hcur.clMap, clusters, hcur.plain); hcur.memo.resize(N, 1.0); hcur.p = calcScore(params, freqMap, hcur.plain, hcur.memo); ++nCur; @@ -742,6 +742,10 @@ namespace Cipher { //printf("Processing cluster %2d ('%c') - count = %d\n", kvSorted.first, getEncodedChar(kvSorted.first), (int) kvSorted.second.size()); + if (hypothesesCur[0].clMap.at(kvSorted.first) != 0) { + continue; + } + int nNew = 0; for (int j = 0; j < nCur; ++j) { const auto & hcur = hypothesesCur[j]; @@ -757,9 +761,6 @@ namespace Cipher { hnew.nused[a]++; for (int k = 0; k < (int) kvSorted.second.size(); ++k) { const auto idx = kvSorted.second[k]; - if ((int) params.hint.size() > idx && params.hint[idx] != -1) { - continue; - } hnew.plain[idx] = a; const auto idx0 = std::max(0, idx - freqMap.len + 1); @@ -1272,29 +1273,10 @@ namespace Cipher { } bool Processor::compute() { - auto clustersNew = m_curResult.clusters; - - for (int iter = 0; iter < m_params.nIters; ++iter) { - clustersNew = m_curResult.clusters; - Cipher::mutateClusters(m_params, clustersNew); - const auto pNew = calcPClusters(m_params, m_similarityMap, m_logMap, m_logMapInv, clustersNew, m_curResult.clMap); - - ++m_nInitialIters; - if (pNew > m_pCur) { - m_curResult.clusters = clustersNew; - m_pCur = pNew; - if (m_pCur > m_pZero) { - m_pZero = m_pCur; - } - - if (m_nInitialIters > m_params.nInitialIters) { - Cipher::beamSearch(m_params, *m_freqMap, m_curResult); - } - } - - m_curResult.pClusters = m_pCur; - ++m_curResult.id; - } + auto clusterings = getClusterings(1); + m_curResult = clusterings[0]; + Cipher::beamSearch(m_params, *m_freqMap, m_curResult); + m_curResult.id++; return true; } diff --git a/subbreak3.h b/subbreak3.h index 3dc62e0..431e09f 100644 --- a/subbreak3.h +++ b/subbreak3.h @@ -40,7 +40,7 @@ namespace Cipher { float wEnglishFreq = 10.0f; // beam search - int nHypothesesToKeep = 100; + int nHypothesesToKeep = 500; THint hint = {}; };