Skip to content

Commit

Permalink
[MINOR] Dependency tasks enable via Log4j
Browse files Browse the repository at this point in the history
  • Loading branch information
Baunsgaard committed Feb 5, 2025
1 parent bea9c96 commit df36b88
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,12 @@
import org.apache.sysds.runtime.DMLRuntimeException;

public class DependencyTask<E> implements Comparable<DependencyTask<?>>, Callable<E> {
public static final boolean ENABLE_DEBUG_DATA = false; // explain task graph
protected static final Log LOG = LogFactory.getLog(DependencyTask.class.getName());
/** debugging dependency tasks only used if LOG.isDebugEnabled */
public List<DependencyTask<?>> _dependencyTasks = null;

private final Callable<E> _task;
protected final List<DependencyTask<?>> _dependantTasks;
public List<DependencyTask<?>> _dependencyTasks = null; // only for debugging
private CompletableFuture<Future<?>> _future;
private int _rdy = 0;
private Integer _priority = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public List<Future<Future<?>>> submitAll(List<? extends Callable<?>> tasks,
public List<Object> submitAllAndWait(List<DependencyTask<?>> dtasks)
throws ExecutionException, InterruptedException {
List<Object> res = new ArrayList<>();
if(DependencyTask.ENABLE_DEBUG_DATA) {
if(LOG.isDebugEnabled()) {
if (dtasks != null && dtasks.size() > 0)
explainTaskGraph(dtasks);
}
Expand Down Expand Up @@ -172,7 +172,7 @@ public static List<DependencyTask<?>> createDependencyTasks(List<? extends Calla
DependencyTask<?> t = ret.get(i);
for(Callable<?> dep : deps) {
DependencyTask<?> dt = map.get(dep);
if(DependencyTask.ENABLE_DEBUG_DATA) {
if(LOG.isDebugEnabled()) {
t._dependencyTasks = t._dependencyTasks == null ? new ArrayList<>() : t._dependencyTasks;
t._dependencyTasks.add(dt);
}
Expand Down Expand Up @@ -226,10 +226,12 @@ public static void explainTaskGraph(List<DependencyTask<?>> tasks) {
sbs[level].append(offsets[level]);
sbs[level].append(entry.getKey().toString()+"\n");
}
System.out.println("EXPlAIN (TASK-GRAPH):");
StringBuilder sb = new StringBuilder("\n");
sb.append("EXPlAIN (TASK-GRAPH):");
for (int i=0; i<sbs.length; i++) {
System.out.println(sbs[i].toString());
sb.append(sbs[i].toString());
}
LOG.debug(sb.toString());

}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.sysds.test.component.frame.transform;

import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
import org.apache.sysds.common.Types.ValueType;
import org.apache.sysds.runtime.frame.data.FrameBlock;
import org.apache.sysds.runtime.transform.encode.CompressedEncode;
import org.apache.sysds.runtime.transform.encode.EncoderFactory;
import org.apache.sysds.runtime.transform.encode.MultiColumnEncoder;
import org.apache.sysds.runtime.util.DependencyTask;
import org.apache.sysds.runtime.util.DependencyThreadPool;
import org.apache.sysds.test.LoggingUtils;
import org.apache.sysds.test.LoggingUtils.TestAppender;
import org.apache.sysds.test.TestUtils;
import org.junit.Test;

public class TransformLogger {
protected static final Log LOG = LogFactory.getLog(TransformLogger.class.getName());

private final FrameBlock data;

public TransformLogger() {
try {

data = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT4}, 231);
data.setSchema(new ValueType[] {ValueType.INT32});
}
catch(Exception e) {
e.printStackTrace();
fail(e.getMessage());
throw e;
}
}

@Test
public void testDummyCode() {
test("{dummycode:[C1]}");
}

public void test(String spec) {
final TestAppender appender = LoggingUtils.overwrite();

try {
Logger.getLogger(CompressedEncode.class).setLevel(Level.DEBUG);
Logger.getLogger(DependencyThreadPool.class).setLevel(Level.DEBUG);
Logger.getLogger(DependencyTask.class).setLevel(Level.DEBUG);

FrameBlock meta = null;
MultiColumnEncoder encoderNormal = EncoderFactory.createEncoder(spec, data.getColumnNames(),
data.getNumColumns(), meta);
encoderNormal.encode(data, 10);

final List<LoggingEvent> log = LoggingUtils.reinsert(appender);

boolean containsMessage = false;
for(LoggingEvent l : log) {
containsMessage |= l.getMessage().toString().contains("EXPlAIN (TASK-GRAPH):");
}

assertTrue(containsMessage);

}
catch(Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
finally {
LoggingUtils.reinsert(appender);
}

}

}

0 comments on commit df36b88

Please sign in to comment.