MarginaliaSearch/code/execution/api/src/main/protobuf/executor-api.proto at master · us3r1d/MarginaliaSearch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
syntax="proto3";

package nu.marginalia.functions.execution.api;

option java_package="nu.marginalia.functions.execution.api";
option java_multiple_files=true;

service ExecutorApi {
  rpc startFsm(RpcFsmName) returns (Empty) {}
  rpc stopFsm(RpcFsmName) returns (Empty) {}
  rpc stopProcess(RpcProcessId) returns (Empty) {}

  rpc getActorStates(Empty) returns (RpcActorRunStates) {}
  rpc listSideloadDir(Empty) returns (RpcUploadDirContents) {}
  rpc listFileStorage(RpcFileStorageId) returns (RpcFileStorageContent) {}

  rpc downloadSampleData(RpcDownloadSampleData) returns (Empty) {}
  rpc calculateAdjacencies(Empty) returns (Empty) {}
  rpc restoreBackup(RpcFileStorageId) returns (Empty) {}

  rpc updateNsfwFilters(RpcUpdateNsfwFilters) returns (Empty) {}

  rpc restartExecutorService(Empty) returns (Empty) {}

  rpc fetchCrawlDataSample(RpcCrawlDataSampleReq) returns (RpcCrawlDataSampleRsp) {}
}

service ExecutorCrawlApi {
  rpc triggerCrawl(RpcFileStorageId) returns (Empty) {}
  rpc triggerSingleDomainRecrawl(RpcFileStorageIdWithDomainName) returns (Empty) {}
  rpc triggerConvert(RpcFileStorageId) returns (Empty) {}
  rpc triggerConvertAndLoad(RpcFileStorageId) returns (Empty) {}
  rpc loadProcessedData(RpcFileStorageIds) returns (Empty) {}
  rpc createCrawlSpecFromDownload(RpcCrawlSpecFromDownload) returns (Empty) {}
}

service ExecutorSideloadApi {
  rpc sideloadEncyclopedia(RpcSideloadEncyclopedia) returns (Empty) {}
  rpc sideloadDirtree(RpcSideloadDirtree) returns (Empty) {}
  rpc sideloadWarc(RpcSideloadWarc) returns (Empty) {}
  rpc sideloadReddit(RpcSideloadReddit) returns (Empty) {}
  rpc sideloadStackexchange(RpcSideloadStackexchange) returns (Empty) {}
}

service ExecutorExportApi {
  rpc exportAtags(RpcExportRequest) returns (Empty) {}
  rpc exportSegmentationModel(RpcExportSegmentationModel) returns (Empty) {}
  rpc exportSampleData(RpcExportSampleData) returns (Empty) {}
  rpc exportDomSampleData(RpcExportDomSampleData) returns (Empty) {}
  rpc exportRssFeeds(RpcExportRequest) returns (Empty) {}
  rpc exportTermFrequencies(RpcExportRequest) returns (Empty) {}
  rpc exportData(Empty) returns (Empty) {}

  rpc exportAllAtags(Empty) returns (Empty) {}
  rpc exportAllFeeds(Empty) returns (Empty) {}
  rpc exportAllTfreqs(Empty) returns (Empty) {}
}

message Empty {}

message RpcFsmName {
  string actorName = 1;
}
message RpcProcessId {
  string processId = 1;
}
message RpcFileStorageId {
  int64 fileStorageId = 1;
}
message RpcExportRequest {
  int64 fileStorageId = 1;
  int64 msgId = 2;
}
message RpcUpdateNsfwFilters {
  int64 msgId = 1;
}
message RpcFileStorageIdWithDomainName {
  int64 fileStorageId = 1;
  string targetDomainName = 2;
}
message RpcFileStorageIds {
  repeated int64 fileStorageIds = 1;
}
message RpcSideloadEncyclopedia {
  string sourcePath = 1;
  string baseUrl = 2;
}
message RpcExportSegmentationModel {
  string sourcePath = 1;
}
message RpcSideloadDirtree {
  string sourcePath = 1;
}
message RpcSideloadWarc {
  string sourcePath = 1;
}
message RpcSideloadReddit {
  string sourcePath = 1;
}
message RpcSideloadStackexchange {
  string sourcePath = 1;
}
message RpcCrawlSpecFromDownload {
  string description = 1;
  string url = 2;
}
message RpcExportSampleData {
  int64 fileStorageId = 1;
  int32 size = 2;
  string name = 3;
  string ctFilter = 4;
}

message RpcExportDomSampleData {
  int64 msgId = 1;
}

message RpcDownloadSampleData {
  string sampleSet = 1;
}
message RpcActorRunStates {
  int32 node = 1;
  repeated RpcActorRunState actorRunStates = 2;
}
message RpcActorRunState {
  string actorName = 1;
  string state = 2;
  string actorDescription = 3;
  string stateDescription = 4;
  bool terminal = 5;
  bool canStart = 6;
}
message RpcUploadDirContents {
  string path = 1;
  repeated RpcUploadDirEntry entries = 2;
}
message RpcUploadDirEntry {
  string name = 1;
  string lastModifiedTime = 2;
  bool isDirectory = 3;
  int64 size = 4;
}
message RpcFileStorageContent {
  repeated RpcFileStorageEntry entries = 1;
}
message RpcFileStorageEntry {
  string name = 1;
  int64 size = 2;
  string lastModifiedTime = 3;
}


message RpcCrawlDataSampleReq {
  int64 fileStorageId = 1;
  string path = 2;

  int32 after = 3;
  string urlGlob = 4;
  string contentType = 5;
  int32 httpStatus = 6;
}

message RpcCrawlDataSampleRsp {
  repeated RpcCrawlDataSampleRecord records = 1;
  repeated RpcCrawlDataSamplesByStatusCode byStatusCode = 2;
  repeated RpcCrawlDataSamplesByContentType byContentType = 3;
  string domain = 4;
}

message RpcCrawlDataSampleRecord {
  string url = 1;
  string contentType = 2;
  int32 httpStatus = 3;
  bool hasBody = 4;
  string etag = 5;
  string lastModified = 6;
}

message RpcCrawlDataSamplesByStatusCode {
  int32 statusCode = 1;
  int32 count = 2;
}

message RpcCrawlDataSamplesByContentType {
  string contentType = 1;
  int32 count = 2;
}