77import com .park .utmstack .domain .chart_builder .types .query .FilterType ;
88import com .park .utmstack .domain .index_pattern .enums .SystemIndexPattern ;
99import com .park .utmstack .repository .UserRepository ;
10+ import com .park .utmstack .domain .notification .NotificationSource ;
11+ import com .park .utmstack .domain .notification .NotificationType ;
1012import com .park .utmstack .service .MailService ;
1113import com .park .utmstack .service .UtmSpaceNotificationControlService ;
1214import com .park .utmstack .service .application_events .ApplicationEventService ;
1315import com .park .utmstack .service .index_policy .IndexPolicyService ;
16+ import com .park .utmstack .service .notification .UtmNotificationService ;
1417import com .park .utmstack .service .dto .compliance .UtmComplianceControlEvaluationHistoryDto ;
1518import com .park .utmstack .service .mapper .compliance .UtmComplianceControlLatestEvaluationMapper ;
1619import com .park .utmstack .service .mapper .compliance .UtmComplianceControlEvaluationHistoryMapper ;
@@ -63,18 +66,21 @@ public class ElasticsearchService {
6366 private final MailService mailService ;
6467 private final UtmSpaceNotificationControlService spaceNotificationControlService ;
6568 private final IndexPolicyService indexPolicyService ;
69+ private final UtmNotificationService notificationService ;
6670 private final OpensearchClientBuilder client ;
6771
6872 public ElasticsearchService (ApplicationEventService eventService , UserRepository userRepository ,
6973 MailService mailService ,
7074 UtmSpaceNotificationControlService spaceNotificationControlService ,
7175 IndexPolicyService indexPolicyService ,
76+ UtmNotificationService notificationService ,
7277 OpensearchClientBuilder client ) {
7378 this .eventService = eventService ;
7479 this .userRepository = userRepository ;
7580 this .mailService = mailService ;
7681 this .spaceNotificationControlService = spaceNotificationControlService ;
7782 this .indexPolicyService = indexPolicyService ;
83+ this .notificationService = notificationService ;
7884 this .client = client ;
7985 }
8086
@@ -241,12 +247,12 @@ public void preventSystemCrashBySpace() {
241247
242248 float diskPercent = clusterStatus .getResume ().getDiskUsedPercent ();
243249
244- if (diskPercent < 70 )
250+ if (diskPercent < 80 )
245251 return ;
246252
247253 if (diskPercent >= 85 ) {
248254 deleteOldestIndices ();
249- } else if (diskPercent >= 70 ) {
255+ } else if (diskPercent >= 80 ) {
250256 List <User > admins = userRepository .findAllAdmins ();
251257 if (CollectionUtils .isEmpty (admins ))
252258 return ;
@@ -261,6 +267,9 @@ public void preventSystemCrashBySpace() {
261267 if (Objects .isNull (notificationControl .getNextNotification ()) ||
262268 now .isAfter (notificationControl .getNextNotification ())) {
263269 mailService .sendLowSpaceEmail (admins , clusterStatus );
270+ notificationService .sendNotification (
271+ String .format ("OpenSearch cluster disk usage at %.1f%%. Oldest log indices will be auto-deleted at 85%%." , diskPercent ),
272+ NotificationSource .SYSTEM , NotificationType .WARNING );
264273 notificationControl .setNextNotification (now .plus (24 , ChronoUnit .HOURS ));
265274 spaceNotificationControlService .save (notificationControl );
266275 }
@@ -278,14 +287,28 @@ public void preventSystemCrashBySpace() {
278287 private void deleteOldestIndices () {
279288 final String ctx = CLASSNAME + ".deleteOldestIndices" ;
280289 try {
281- List <IndicesRecord > indices = client .getClient ().getIndices (Constants .SYS_INDEX_PATTERN .get (SystemIndexPattern .LOGS ), IndexSort .builder ()
282- .with (IndexSortableProperty .CreationDate , SortOrder .Asc ).build ());
290+ List <String > patterns = Arrays .asList (
291+ Constants .SYS_INDEX_PATTERN .get (SystemIndexPattern .LOGS ),
292+ "security-auditlog-*" ,
293+ "top_queries-*" );
294+ IndexSort sortAsc = IndexSort .builder ()
295+ .with (IndexSortableProperty .CreationDate , SortOrder .Asc ).build ();
296+
297+ List <IndicesRecord > indices = new ArrayList <>();
298+ for (String pattern : patterns ) {
299+ try {
300+ indices .addAll (client .getClient ().getIndices (pattern , sortAsc ));
301+ } catch (Exception e ) {
302+ log .warn ("{}: pattern {} lookup failed: {}" , ctx , pattern , e .getMessage ());
303+ }
304+ }
305+
306+ indices .sort (Comparator .comparing (IndicesRecord ::creationDateString , Comparator .nullsLast (String ::compareTo )));
283307
284- // If no index that match with log-* was found then te function is terminated
285308 if (CollectionUtils .isEmpty (indices ))
286309 return ;
287310
288- // Indices are returned from oldest to newest ordered by creation.date asc
311+ // Indices are ordered from oldest to newest by creation.date asc
289312 for (IndicesRecord index : indices ) {
290313 Optional <ElasticCluster > opt = getClusterStatus ();
291314
0 commit comments