Skip to content

fix(toolkit): keep source DB intact when DbMove copy fails #140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
halibobo1205 wants to merge 1 commit into from
+351 −67
Closed

fix(toolkit): keep source DB intact when DbMove copy fails #140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions plugins/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

This package contains a set of tools for TRON, the followings are the documentation for each tool.

NOTE: All `db` tools operate directly on the database files. Before performing a database
operation (archive, convert, copy, lite, mv, root), you must stop the currently running
FullNode service.

## DB Archive(Requires x86 + LevelDB)

DB archive provides the ability to reformat the manifest according to the current `database`, parameters are compatible with the previous `ArchiveManifest`.
Expand Down
3 changes: 3 additions & 0 deletions plugins/src/main/java/common/org/tron/plugins/Db.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
mixinStandardHelpOptions = true,
version = "db command 1.0",
description = "An rich command set that provides high-level operations for dbs.",
header = "All `db` tools operate directly on the database files.\n"
+ "Before performing a database operation,\n"
+ "you must stop the currently running FullNode service.\n",
subcommands = {CommandLine.HelpCommand.class,
DbMove.class,
DbArchive.class,
Expand Down
188 changes: 127 additions & 61 deletions plugins/src/main/java/common/org/tron/plugins/DbMove.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,21 @@
import com.typesafe.config.ConfigFactory;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.Arrays;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import me.tongfei.progressbar.ProgressBar;
import org.tron.plugins.utils.FileUtils;
Expand Down Expand Up @@ -76,70 +80,138 @@ public Integer call() throws Exception {
printNotExist();
return 0;
}
List<Property> toBeMove = dbs.stream()
.map(c -> {
try {
return new Property(c.getString(NAME_CONFIG_KEY),
Paths.get(database.toString(), dbPath, c.getString(NAME_CONFIG_KEY)),
Paths.get(c.getString(PATH_CONFIG_KEY), dbPath, c.getString(NAME_CONFIG_KEY)));
} catch (IOException e) {
spec.commandLine().getErr().println(e);
}
return null;
}).filter(Objects::nonNull)
.filter(p -> !p.destination.equals(p.original)).collect(Collectors.toList());

if (toBeMove.isEmpty()) {
printNotExist();
return 0;
List<Property> toBeMove = new ArrayList<>();
for (Config c : dbs) {
try {
toBeMove.add(new Property(c.getString(NAME_CONFIG_KEY),
Paths.get(database.toString(), dbPath, c.getString(NAME_CONFIG_KEY)),
Paths.get(c.getString(PATH_CONFIG_KEY), dbPath, c.getString(NAME_CONFIG_KEY))));
} catch (IOException e) {
spec.commandLine().getErr().println(e);
return 2;
}
}
boolean allCopied = ProgressBar.wrap(toBeMove.stream(), "copy task")
.allMatch(this::copy);
if (!allCopied) {
cleanupDestinations(toBeMove);
return 1;
}
toBeMove = toBeMove.stream()
.filter(property -> {
if (property.destination.toFile().exists()) {
spec.commandLine().getOut().println(String.format("%s already exist,skip.",
property.destination));
return false;
} else {
return true;
}
}).collect(Collectors.toList());

if (toBeMove.isEmpty()) {
printNotExist();
return 0;
boolean allMoved = ProgressBar.wrap(toBeMove.stream(), "link task")
.map(this::replaceSourceWithLink).reduce(Boolean.TRUE, Boolean::logicalAnd);
if (!allMoved) {
return 1;
}
ProgressBar.wrap(toBeMove.stream(), "mv task").forEach(this::run);
spec.commandLine().getOut().println("move db done.");

} else {
printNotExist();
return 0;
}
return 0;
}

private void run(Property p) {
if (p.destination.toFile().mkdirs()) {
ProgressBar.wrap(Arrays.stream(Objects.requireNonNull(p.original.toFile().listFiles()))
.filter(File::isFile).map(File::getName).parallel(), p.name).forEach(file -> {
Path original = Paths.get(p.original.toString(), file);
Path destination = Paths.get(p.destination.toString(), file);
try {
Files.copy(original, destination,
StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
spec.commandLine().getErr().println(e);
}
});
private boolean copy(Property p) {
if (!p.destination.toFile().mkdirs()) {
spec.commandLine().getErr().println(String.format("%s create failed.", p.destination));
return false;
}

AtomicBoolean hasError = new AtomicBoolean(false);
try (Stream<Path> files = Files.walk(p.original)) {

@chatgpt-codex-connector chatgpt-codex-connector Bot Sep 3, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Reject destinations nested under their source

When a configured destination resolves beneath the source database, mkdirs() creates that destination before this recursive walk begins, so the walk can discover the directories that copyEntry() is generating and repeatedly copy the destination into itself until a path-length or disk-space failure occurs. Validate that p.destination does not start with p.original before starting the recursive copy.

Useful? React with 👍 / 👎.

ProgressBar.wrap(files.parallel(), p.name).forEach(source -> {
if (hasError.get()) {
return;
}
try {
copyEntry(p, source);
} catch (IOException e) {
hasError.set(true);
Comment thread
halibobo1205 marked this conversation as resolved.
spec.commandLine().getErr().println(e);
}
});
} catch (IOException | UncheckedIOException e) {
hasError.set(true);
spec.commandLine().getErr().println(e);
}

if (hasError.get()) {
spec.commandLine().getErr().println(String.format(
"%s copy to %s failed, source kept.",
p.original, p.destination));
return false;
}
return true;
}

private void copyEntry(Property p, Path source) throws IOException {
BasicFileAttributes attributes = Files.readAttributes(
source, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
Path destination = p.destination.resolve(p.original.relativize(source));
if (attributes.isDirectory()) {
Files.createDirectories(destination);
} else if (attributes.isRegularFile()) {
Files.createDirectories(destination.getParent());
Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING);
} else {
throw new IOException(String.format(
"%s is neither a regular file nor a directory, can not be moved.", source));
}
}

private boolean replaceSourceWithLink(Property p) {
try {
if (!FileUtils.deleteDir(p.original.toFile())) {
spec.commandLine().getErr().println(String.format(
"%s delete failed and may be incomplete; the only complete copy is at %s, keep it.",
p.original, p.destination));
printRecoveryHint(p);
return false;
}
Files.createSymbolicLink(p.original, p.destination);
return true;
} catch (IOException | RuntimeException x) {
spec.commandLine().getErr().println(x);
spec.commandLine().getErr().println(String.format(
"%s move failed; the complete copy is at %s, keep it.",
p.original, p.destination));
printRecoveryHint(p);
return false;
}
}

private void printRecoveryHint(Property p) {
spec.commandLine().getErr().println(String.format(
"To recover manually: remove %s if present, then create a symbolic link at %s"
+ " pointing to %s.",
p.original, p.original, p.destination));
}

private void cleanupDestinations(List<Property> properties) {
boolean allCleaned = properties.stream().map(property -> {
File destination = property.destination.toFile();
if (Files.notExists(destination.toPath(), LinkOption.NOFOLLOW_LINKS)) {
return true;
}
try {
if (FileUtils.deleteDir(p.original.toFile())) {
Files.createSymbolicLink(p.original, p.destination);
if (FileUtils.deleteDir(destination)) {
Comment thread
halibobo1205 marked this conversation as resolved.
return true;
}
} catch (IOException | UnsupportedOperationException x) {
spec.commandLine().getErr().println(x);
} catch (RuntimeException e) {
spec.commandLine().getErr().println(e);
}
spec.commandLine().getErr().println(String.format(
"%s cleanup failed; remove the leftover copy before retrying.",
property.destination));
return false;
}).reduce(Boolean.TRUE, Boolean::logicalAnd);

if (allCleaned) {
spec.commandLine().getErr().println(
"move db failed; all source databases were kept, please retry.");
} else {
spec.commandLine().getErr().println(String.format("%s create failed.", p.destination));
spec.commandLine().getErr().println(
"move db failed; all source databases were kept, but leftover copies remain.");
}
}

Expand Down Expand Up @@ -167,7 +239,7 @@ public Property(String name, Path original, Path destination) throws IOException
throw new IOException(original + " is symbolicLink!");
}
this.destination = destination.toFile().getCanonicalFile().toPath();
if (this.destination.toFile().exists()) {
if (!Files.notExists(this.destination, LinkOption.NOFOLLOW_LINKS)) {
throw new IOException(this.destination + " already exist!");
}
if (this.destination.equals(this.original)) {
Expand Down Expand Up @@ -195,9 +267,6 @@ public Config convert(String value) throws Exception {
if (dbs.isEmpty()) {
throw notFind;
}
String dbPath = config.hasPath(DB_DIRECTORY_CONFIG_KEY)
? config.getString(DB_DIRECTORY_CONFIG_KEY) : DEFAULT_DB_DIRECTORY;

dbs = dbs.stream()
.filter(c -> c.hasPath(NAME_CONFIG_KEY) && c.hasPath(PATH_CONFIG_KEY))
.collect(Collectors.toList());
Expand All @@ -207,13 +276,10 @@ public Config convert(String value) throws Exception {
}
Set<String> toBeMove = new HashSet<>();
for (Config c : dbs) {
if (!toBeMove.add(new Property(c.getString(NAME_CONFIG_KEY),
Paths.get(database.toString(), dbPath, c.getString(NAME_CONFIG_KEY)),
Paths.get(c.getString(PATH_CONFIG_KEY), dbPath,
c.getString(NAME_CONFIG_KEY))).name)) {
String name = c.getString(NAME_CONFIG_KEY);
if (!toBeMove.add(name)) {
throw new IllegalArgumentException(
"DB config has duplicate key:[" + c.getString(NAME_CONFIG_KEY)
+ "],please check! ");
"DB config has duplicate key:[" + name + "],please check! ");
}
}
} else {
Expand Down
Loading
Loading