Skip to content

Commit 1f140e0

Browse files
monatismonatis
authored
Introduce common-clip lib for utilities (monatis#28)
1 parent 6143d69 commit 1f140e0

File tree

9 files changed

+237
-196
lines changed

9 files changed

+237
-196
lines changed

‎clip.cpp‎

Lines changed: 0 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1477,79 +1477,3 @@ bool image_normalize(clip_image_u8 *img, clip_image_f32 *res)
14771477
}
14781478
return true;
14791479
}
1480-
1481-
// utility functions mainly intended for examples and debugging
1482-
1483-
bool app_params_parse(int argc, char **argv, app_params &params)
1484-
{
1485-
for (int i = 0; i < argc; i++)
1486-
{
1487-
std::string arg = std::string(argv[i]);
1488-
if (arg == "-m" || arg == "--model")
1489-
{
1490-
params.model = argv[++i];
1491-
}
1492-
else if (arg == "-t" || arg == "--threads")
1493-
{
1494-
params.n_threads = std::stoi(argv[++i]);
1495-
}
1496-
else if (arg == "--text")
1497-
{
1498-
params.texts.push_back(argv[++i]);
1499-
}
1500-
else if (arg == "--image")
1501-
{
1502-
params.image_paths.push_back(argv[++i]);
1503-
}
1504-
else if (arg == "-v" || arg == "--verbose")
1505-
{
1506-
params.verbose = std::stoi(argv[++i]);
1507-
}
1508-
else if (arg == "-h" || arg == "--help")
1509-
{
1510-
print_help(argc, argv, params);
1511-
exit(0);
1512-
}
1513-
else
1514-
{
1515-
if (i != 0)
1516-
{
1517-
printf("%s: unrecognized argument: %s\n", __func__, arg.c_str());
1518-
return false;
1519-
}
1520-
}
1521-
}
1522-
return params.image_paths.size() >= 1 && params.texts.size() >= 1;
1523-
}
1524-
1525-
void print_help(int argc, char **argv, app_params &params)
1526-
{
1527-
printf("Usage: %s [options]\n", argv[0]);
1528-
printf("\nOptions:");
1529-
printf(" -h, --help: Show this message and exit\n");
1530-
printf(" -m <path>, --model <path>: path to model. Default: %s\n", params.model.c_str());
1531-
printf(" -t N, --threads N: Number of threads to use for inference. Default: %d\n", params.n_threads);
1532-
printf(" --text <text>: Text to encode. At least one text should be specified\n");
1533-
printf(" --image <path>: Path to an image file. At least one image path should be specified\n");
1534-
printf(" -v <level>, --verbose <level>: Control the level of verbosity. 0 = minimum, 2 = maximum. Default: %d\n", params.verbose);
1535-
}
1536-
1537-
void write_floats_to_file(float *array, int size, char *filename)
1538-
{
1539-
// Open the file for writing.
1540-
FILE *file = fopen(filename, "w");
1541-
if (file == NULL)
1542-
{
1543-
printf("Error opening file: %s\n", filename);
1544-
return;
1545-
}
1546-
1547-
// Write the float values to the file.
1548-
for (int i = 0; i < size; i++)
1549-
{
1550-
fprintf(file, "%f\n", array[i]);
1551-
}
1552-
1553-
// Close the file.
1554-
fclose(file);
1555-
}

‎clip.h‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,6 @@
1212
// extern "C" {
1313
// #endif
1414

15-
struct app_params
16-
{
17-
int32_t n_threads = std::min(4, (int32_t)std::thread::hardware_concurrency());
18-
19-
std::string model = "models/ggml-model-f16.bin";
20-
std::vector<std::string> image_paths;
21-
std::vector<std::string> texts;
22-
int verbose = 1;
23-
};
24-
25-
bool app_params_parse(int argc, char **argv, app_params &params);
26-
void print_help(int argc, char **argv, app_params &params);
27-
2815
// default hparams for text_model (ViT-B/32)
2916
struct clip_text_hparams
3017
{

‎examples/CMakeLists.txt‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
1+
2+
add_library(common-clip STATIC common-clip.cpp)
3+
target_link_libraries(common-clip PRIVATE ggml)
4+
target_include_directories(common-clip PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
5+
16
add_executable(main main.cpp)
2-
target_link_libraries(main PRIVATE clip ggml)
7+
target_link_libraries(main PRIVATE clip common-clip ggml)
38

49
add_executable(zsl zsl.cpp)
5-
target_link_libraries(zsl PRIVATE clip ggml)
10+
target_link_libraries(zsl PRIVATE clip common-clip ggml)

‎examples/common-clip.cpp‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
#include "common-clip.h"
2+
3+
#include <cassert>
4+
#include <cstdlib>
5+
#include <cstring>
6+
#include <cmath>
7+
#include <iostream>
8+
#include <fstream>
9+
#include <vector>
10+
#include <map>
11+
12+
#ifdef _WIN32
13+
#include <windows.h>
14+
#include <tchar.h>
15+
#else
16+
#include <dirent.h>
17+
#include <sys/stat.h>
18+
#endif
19+
20+
// common utility functions mainly intended for examples and debugging
21+
22+
std::map<std::string, std::vector<std::string>> get_dir_keyed_files(const std::string &path, uint32_t max_files_per_dir = 0)
23+
{
24+
std::map<std::string, std::vector<std::string>> result;
25+
26+
#ifdef _WIN32
27+
std::string wildcard = path + "\\*";
28+
WIN32_FIND_DATAA fileData;
29+
HANDLE hFind = FindFirstFileA(wildcard.c_str(), &fileData);
30+
31+
if (hFind == INVALID_HANDLE_VALUE)
32+
{
33+
std::cerr << "Failed to open directory: " << path << std::endl;
34+
return result;
35+
}
36+
37+
uint32_t fileCount = 0;
38+
39+
do
40+
{
41+
std::string name = fileData.cFileName;
42+
std::string fullPath = path + "\\" + name;
43+
44+
if (fileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
45+
{
46+
// Skip . and ..
47+
if (name == "." || name == "..")
48+
continue;
49+
50+
std::map<std::string, std::vector<std::string>> subResult = get_dir_keyed_files(fullPath, max_files_per_dir);
51+
result.insert(subResult.begin(), subResult.end());
52+
}
53+
else
54+
{
55+
size_t pos = path.find_last_of("\\/");
56+
std::string parentDir = (pos != std::string::npos) ? path.substr(pos + 1) : path;
57+
result[parentDir].push_back(fullPath);
58+
59+
++fileCount;
60+
if (max_files_per_dir > 0 && fileCount >= max_files_per_dir)
61+
break;
62+
}
63+
} while (FindNextFileA(hFind, &fileData));
64+
65+
FindClose(hFind);
66+
#else
67+
DIR *dir;
68+
struct dirent *entry;
69+
struct stat fileStat;
70+
71+
if ((dir = opendir(path.c_str())) == NULL)
72+
{
73+
std::cerr << "Failed to open directory: " << path << std::endl;
74+
return result;
75+
}
76+
77+
uint32_t fileCount = 0;
78+
79+
while ((entry = readdir(dir)) != NULL)
80+
{
81+
std::string name = entry->d_name;
82+
std::string fullPath = path + "/" + name;
83+
84+
if (stat(fullPath.c_str(), &fileStat) < 0)
85+
{
86+
std::cerr << "Failed to get file stat: " << fullPath << std::endl;
87+
continue;
88+
}
89+
90+
if (S_ISDIR(fileStat.st_mode))
91+
{
92+
// Skip . and ..
93+
if (name == "." || name == "..")
94+
continue;
95+
96+
std::map<std::string, std::vector<std::string>> subResult = get_dir_keyed_files(fullPath, max_files_per_dir);
97+
result.insert(subResult.begin(), subResult.end());
98+
}
99+
else
100+
{
101+
size_t pos = path.find_last_of("/");
102+
std::string parentDir = (pos != std::string::npos) ? path.substr(pos + 1) : path;
103+
result[parentDir].push_back(fullPath);
104+
105+
++fileCount;
106+
if (max_files_per_dir > 0 && fileCount >= max_files_per_dir)
107+
break;
108+
}
109+
}
110+
111+
closedir(dir);
112+
#endif
113+
114+
return result;
115+
}
116+
117+
bool app_params_parse(int argc, char **argv, app_params &params)
118+
{
119+
for (int i = 0; i < argc; i++)
120+
{
121+
std::string arg = std::string(argv[i]);
122+
if (arg == "-m" || arg == "--model")
123+
{
124+
params.model = argv[++i];
125+
}
126+
else if (arg == "-t" || arg == "--threads")
127+
{
128+
params.n_threads = std::stoi(argv[++i]);
129+
}
130+
else if (arg == "--text")
131+
{
132+
params.texts.push_back(argv[++i]);
133+
}
134+
else if (arg == "--image")
135+
{
136+
params.image_paths.push_back(argv[++i]);
137+
}
138+
else if (arg == "-v" || arg == "--verbose")
139+
{
140+
params.verbose = std::stoi(argv[++i]);
141+
}
142+
else if (arg == "-h" || arg == "--help")
143+
{
144+
print_help(argc, argv, params);
145+
exit(0);
146+
}
147+
else
148+
{
149+
if (i != 0)
150+
{
151+
printf("%s: unrecognized argument: %s\n", __func__, arg.c_str());
152+
return false;
153+
}
154+
}
155+
}
156+
return params.image_paths.size() >= 1 && params.texts.size() >= 1;
157+
}
158+
159+
void print_help(int argc, char **argv, app_params &params)
160+
{
161+
printf("Usage: %s [options]\n", argv[0]);
162+
printf("\nOptions:");
163+
printf(" -h, --help: Show this message and exit\n");
164+
printf(" -m <path>, --model <path>: path to model. Default: %s\n", params.model.c_str());
165+
printf(" -t N, --threads N: Number of threads to use for inference. Default: %d\n", params.n_threads);
166+
printf(" --text <text>: Text to encode. At least one text should be specified\n");
167+
printf(" --image <path>: Path to an image file. At least one image path should be specified\n");
168+
printf(" -v <level>, --verbose <level>: Control the level of verbosity. 0 = minimum, 2 = maximum. Default: %d\n", params.verbose);
169+
}
170+
171+
void write_floats_to_file(float *array, int size, char *filename)
172+
{
173+
// Open the file for writing.
174+
FILE *file = fopen(filename, "w");
175+
if (file == NULL)
176+
{
177+
printf("Error opening file: %s\n", filename);
178+
return;
179+
}
180+
181+
// Write the float values to the file.
182+
for (int i = 0; i < size; i++)
183+
{
184+
fprintf(file, "%f\n", array[i]);
185+
}
186+
187+
// Close the file.
188+
fclose(file);
189+
}

‎examples/common-clip.h‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#ifndef COMMON_CLIP_H
2+
#define COMMON_CLIP_H
3+
4+
#include <vector>
5+
#include <map>
6+
#include <cstring>
7+
#include <thread>
8+
9+
// #ifdef __cplusplus
10+
// extern "C" {
11+
// #endif
12+
13+
std::map<std::string, std::vector<std::string>> get_dir_keyed_files(const std::string &path, uint32_t max_files_per_dir);
14+
15+
struct app_params
16+
{
17+
int32_t n_threads = std::min(4, (int32_t)std::thread::hardware_concurrency());
18+
19+
std::string model = "models/ggml-model-f16.bin";
20+
std::vector<std::string> image_paths;
21+
std::vector<std::string> texts;
22+
int verbose = 1;
23+
};
24+
25+
bool app_params_parse(int argc, char **argv, app_params &params);
26+
void print_help(int argc, char **argv, app_params &params);
27+
28+
// utils for debugging
29+
void write_floats_to_file(float *array, int size, char *filename);
30+
31+
// #ifdef __cplusplus
32+
// }
33+
// #endif
34+
35+
#endif // COMMON_CLIP_H

‎examples/main.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1+
// main example to demonstrate usage of the API
2+
13
#include "clip.h"
4+
#include "common-clip.h"
25

36
int main(int argc, char **argv)
47
{

‎examples/zsl.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// zero-shot image labeling example
22

33
#include "clip.h"
4+
#include "common-clip.h"
45

56
int main(int argc, char **argv)
67
{

‎tests/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
add_executable(benchmark benchmark.cpp)
2-
target_link_libraries(benchmark PRIVATE clip ggml)
2+
target_link_libraries(benchmark PRIVATE clip common-clip ggml)

0 commit comments

Comments
 (0)