diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java index 4c65621b0fc..21ce1698e05 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java @@ -20,6 +20,8 @@ import static java.nio.charset.StandardCharsets.ISO_8859_1; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -93,6 +95,19 @@ public boolean canAccess(byte[] expression) { } } + public static BytesEvaluator newEvaluator(Collection authsSet) { + Collection> convertedAuths = new ArrayList<>(); + for (Authorizations auths : authsSet) { + List bytesAuths = auths.getAuthorizations(); + Set stringAuths = new HashSet<>(bytesAuths.size()); + for (var auth : bytesAuths) { + stringAuths.add(new String(auth, ISO_8859_1)); + } + convertedAuths.add(stringAuths); + } + return new BytesEvaluator(ACCESS.newEvaluator(convertedAuths)); + } + public static BytesEvaluator newEvaluator(Authorizations auths) { List bytesAuths = auths.getAuthorizations(); Set stringAuths = new HashSet<>(bytesAuths.size()); diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java index 92e5079ab65..4e57838224c 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java @@ -21,6 +21,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; import java.util.Map; import org.apache.accumulo.access.InvalidAccessExpressionException; @@ -44,12 +46,14 @@ */ public class VisibilityFilter extends Filter implements OptionDescriber { + private static final Logger log = LoggerFactory.getLogger(VisibilityFilter.class); + private static final BytesAccess.BytesEvaluator EMPTY_EVALUATOR = + BytesAccess.newEvaluator(Authorizations.EMPTY); + private BytesAccess.BytesEvaluator accessEvaluator; protected Map cache; private final ArrayByteSequence testVis = new ArrayByteSequence(new byte[0]); - private static final Logger log = LoggerFactory.getLogger(VisibilityFilter.class); - private static final String AUTHS = "auths"; private static final String FILTER_INVALID_ONLY = "filterInvalid"; @@ -64,10 +68,29 @@ public void init(SortedKeyValueIterator source, Map op if (!filterInvalid) { String auths = options.get(AUTHS); - Authorizations authObj = auths == null || auths.isEmpty() ? new Authorizations() - : new Authorizations(auths.getBytes(UTF_8)); - - this.accessEvaluator = BytesAccess.newEvaluator(authObj); + if (auths == null || auths.isEmpty()) { + this.accessEvaluator = EMPTY_EVALUATOR; + } else if (!auths.contains(Authorizations.HEADER)) { + this.accessEvaluator = BytesAccess.newEvaluator(new Authorizations(auths.getBytes(UTF_8))); + } else { + String[] authParts = auths.split(Authorizations.HEADER); + if (authParts.length == 0) { + this.accessEvaluator = EMPTY_EVALUATOR; + } else { + Collection authSet = new ArrayList<>(); + for (int i = 0; i < authParts.length; i++) { + String part = authParts[i]; + if (part.isEmpty()) { + continue; + } else { + // split removes the HEADER, need to add it back + String serializedAuthString = Authorizations.HEADER + authParts[i]; + authSet.add(new Authorizations(serializedAuthString.getBytes(UTF_8))); + } + } + this.accessEvaluator = BytesAccess.newEvaluator(authSet); + } + } } this.cache = new LRUMap<>(1000); } @@ -132,13 +155,18 @@ public IteratorOptions describeOptions() { IteratorOptions io = super.describeOptions(); io.setName("visibilityFilter"); io.setDescription("The VisibilityFilter allows you to filter for key/value" - + " pairs by a set of authorizations or filter invalid labels from corrupt files."); + + " pairs by a set(s) of authorizations or filter invalid labels from corrupt files."); io.addNamedOption(FILTER_INVALID_ONLY, "if 'true', the iterator is instructed to ignore the authorizations and" + " only filter invalid visibility labels (default: false)"); - io.addNamedOption(AUTHS, - "the serialized set of authorizations to filter against (default: empty" - + " string, accepts only entries visible by all)"); + io.addNamedOption(AUTHS, "The set of Authorizations to use when filtering (default: empty" + + " string, accepts only entries visible by all). The value can" + + " either be of the older form (\"auth1,auth2\") which only supports" + + " a single Authorizations object, or the newer form which uses " + Authorizations.HEADER + + " concatenated with Base64 encoded comma-separated authorization tokens." + + "The latter case supports one or more Authorizations (\"" + Authorizations.HEADER + + "Base64(auth1,auth2)\") or (\"" + Authorizations.HEADER + "Base64(auth1,auth2)" + + Authorizations.HEADER + "auth2,auth3\")"); return io; } @@ -146,6 +174,12 @@ public static void setAuthorizations(IteratorSetting setting, Authorizations aut setting.addOption(AUTHS, auths.serialize()); } + public static void setAuthorizations(IteratorSetting setting, Collection auths) { + StringBuilder builder = new StringBuilder(); + auths.forEach(a -> builder.append(a.serialize())); + setting.addOption(AUTHS, builder.toString()); + } + public static void filterInvalidLabelsOnly(IteratorSetting setting, boolean featureEnabled) { setting.addOption(FILTER_INVALID_ONLY, Boolean.toString(featureEnabled)); } diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java index 075e2bef330..7ea46d0c2de 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.TreeMap; @@ -155,6 +156,22 @@ public void testAllowAuthorizedLabelsOnly() throws IOException { verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000); } + @Test + public void testMulitAllowAuthorizedLabelsOnly() throws IOException { + IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class); + VisibilityFilter.setAuthorizations(is, + List.of(new Authorizations("abc"), new Authorizations("def"))); + + TreeMap source = createSourceWithHiddenData(1, 2); + verify(source, 3, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1); + + source = createSourceWithHiddenData(30, 500); + verify(source, 530, is.getOptions(), GOOD, GOOD, GOOD_VIS, 30); + + source = createSourceWithHiddenData(1000, 500); + verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000); + } + @Test public void testAllowUnauthorizedLabelsOnly() throws IOException { IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class); @@ -171,6 +188,23 @@ public void testAllowUnauthorizedLabelsOnly() throws IOException { verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500); } + @Test + public void testMultiAllowUnauthorizedLabelsOnly() throws IOException { + IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class); + VisibilityFilter.setNegate(is, true); + VisibilityFilter.setAuthorizations(is, + List.of(new Authorizations("abc"), new Authorizations("def"))); + + TreeMap source = createSourceWithHiddenData(1, 2); + verify(source, 3, is.getOptions(), BAD, BAD, HIDDEN_VIS, 2); + + source = createSourceWithHiddenData(30, 500); + verify(source, 530, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500); + + source = createSourceWithHiddenData(1000, 500); + verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500); + } + @Test public void testNoLabels() throws IOException { IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class); @@ -256,4 +290,11 @@ public void testDeepCopyAfterInit() throws IOException { assertTrue(copyFilter.accept(k, new Value())); } + @Test + public void testAuthorizationSerialization() { + String auth = new Authorizations("abc", "def").serialize(); + System.out.println(auth); + + } + } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java b/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java index 40fe8b4c87e..c6fc0c57e9f 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java @@ -37,12 +37,14 @@ import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.VisibilityFilter; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.util.ByteArraySet; @@ -92,6 +94,7 @@ public void run() throws Exception { insertData(c, table); queryData(c, table); + queryDataMultiAuth(c, table); deleteData(c, table); insertDefaultData(c, table2); @@ -222,6 +225,48 @@ private void queryData(AccumuloClient c, String tableName) throws Exception { queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss(), expected); } + /** + * Configures Scanners with the users default authorizations, then it adds a + * MultiAuthVisibilityFilter with different sets of Authorizations + */ + private void queryDataMultiAuth(AccumuloClient c, String tableName) throws Exception { + + c.securityOperations().changeUserAuthorizations(getAdminPrincipal(), + new Authorizations("A", "B", "FOO", "L", "M", "Z")); + + Authorizations userAuths = c.securityOperations().getUserAuthorizations(c.whoami()); + + Set expectedUserAuths = + Set.of("v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13"); + try (Scanner scanner = c.createScanner(tableName, userAuths); + BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) { + verify(scanner.iterator(), expectedUserAuths.toArray(new String[] {})); + + bs.setRanges(Collections.singleton(new Range())); + verify(bs.iterator(), expectedUserAuths.toArray(new String[] {})); + } + + Authorizations entity1 = new Authorizations("A", "B", "FOO", "L", "M"); + Authorizations entity2 = new Authorizations("B", "FOO", "Z"); + // should only see entries with no column visibility, B and/or FOO + Set expectedAuths = Set.of("v1", "v3", "v11"); + + IteratorSetting is = new IteratorSetting(100, "userAuths", VisibilityFilter.class); + VisibilityFilter.setAuthorizations(is, Set.of(entity1, entity2)); + + try (Scanner scanner = c.createScanner(tableName, userAuths); + BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) { + + scanner.addScanIterator(is); + verify(scanner.iterator(), expectedAuths.toArray(new String[] {})); + + bs.setRanges(Collections.singleton(new Range())); + bs.addScanIterator(is); + verify(bs.iterator(), expectedAuths.toArray(new String[] {})); + } + + } + private void queryData(AccumuloClient c, String tableName, Set allAuths, Set userAuths, Map,Set> expected) throws Exception {